From 32f25d2f2095c13110640f27c99d0c95f2d49255 Mon Sep 17 00:00:00 2001 From: Tomas Farago Date: Wed, 1 Dec 2021 16:31:11 +0100 Subject: read: add image-start and image-step properties to skip images in multi-image files. Fixes #164. --- docs/generators.rst | 8 +++ src/readers/ufo-edf-reader.c | 31 ++++++--- src/readers/ufo-hdf5-reader.c | 13 +++- src/readers/ufo-raw-reader.c | 16 ++++- src/readers/ufo-reader.c | 10 +-- src/readers/ufo-reader.h | 12 ++-- src/readers/ufo-tiff-reader.c | 36 +++++++++-- src/ufo-read-task.c | 142 ++++++++++++++++++++++++++---------------- 8 files changed, 190 insertions(+), 78 deletions(-) diff --git a/docs/generators.rst b/docs/generators.rst index 7e52fb0..9f063ab 100644 --- a/docs/generators.rst +++ b/docs/generators.rst @@ -34,10 +34,18 @@ File reader First index from where files are read. + .. gobj:prop:: image-start:uint + + First image index from where images are read in multi-image files. + .. gobj:prop:: step:uint Number of files to skip. + .. gobj:prop:: image-step:uint + + Number of images to skip in a multi-image file. + .. gobj:prop:: y:uint Vertical coordinate from where to start reading. diff --git a/src/readers/ufo-edf-reader.c b/src/readers/ufo-edf-reader.c index cc52ef1..77d255b 100644 --- a/src/readers/ufo-edf-reader.c +++ b/src/readers/ufo-edf-reader.c @@ -26,6 +26,7 @@ struct _UfoEdfReaderPrivate { FILE *fp; + guint start; gssize size; gsize height; guint bytes_per_sample; @@ -68,6 +69,7 @@ ufo_edf_reader_open (UfoReader *reader, fseek (priv->fp, 0L, SEEK_END); priv->size = (gsize) ftell (priv->fp); fseek (priv->fp, 0L, SEEK_SET); + priv->start = start; return TRUE; } @@ -93,19 +95,22 @@ ufo_edf_reader_data_available (UfoReader *reader) return priv->fp != NULL && ftell (priv->fp) < priv->size; } -static void +static gsize ufo_edf_reader_read (UfoReader *reader, UfoBuffer *buffer, UfoRequisition *requisition, guint roi_y, guint roi_height, - guint roi_step) + guint roi_step, + guint image_step) { UfoEdfReaderPrivate *priv; gsize num_bytes; gsize num_read; gssize offset; gchar *data; + gsize to_skip; + guint start = 0; priv = UFO_EDF_READER_GET_PRIVATE (reader); data = (gchar *) ufo_buffer_get_host_array (buffer, NULL); @@ -113,12 +118,16 @@ ufo_edf_reader_read (UfoReader *reader, /* size of the image width in bytes */ const gsize width = requisition->dims[0] * priv->bytes_per_sample; const guint num_rows = requisition->dims[1]; + if (priv->start) { + start = priv->start; + priv->start = 0; + } const gsize end_position = ftell (priv->fp) + priv->height * width; offset = 0; - /* Go to the first desired row */ - fseek (priv->fp, roi_y * width, SEEK_CUR); + /* Go to the first desired row at *start* image index */ + fseek (priv->fp, start * priv->height * width + roi_y * width, SEEK_CUR); if (roi_step == 1) { /* Read the full ROI at once if no stepping is specified */ @@ -126,14 +135,14 @@ ufo_edf_reader_read (UfoReader *reader, num_read = fread (data, 1, num_bytes, priv->fp); if (num_read != num_bytes) - return; + return 0; } else { for (guint i = 0; i < num_rows - 1; i++) { num_read = fread (data + offset, 1, width, priv->fp); if (num_read != width) - return; + return 0; offset += width; fseek (priv->fp, (roi_step - 1) * width, SEEK_CUR); @@ -144,12 +153,16 @@ ufo_edf_reader_read (UfoReader *reader, num_read = fread (data + offset, 1, width, priv->fp); if (num_read != width) - return; + return 0; } /* Go to the image end to be in a consistent state for the next read */ fseek (priv->fp, end_position, SEEK_SET); + /* Skip the desired number of images */ + to_skip = MIN (image_step - 1, (priv->size - (gsize) ftell (priv->fp)) / (priv->height * width)); + fseek (priv->fp, to_skip * priv->height * width, SEEK_CUR); + if ((G_BYTE_ORDER == G_LITTLE_ENDIAN) && priv->big_endian) { guint32 *conv = (guint32 *) ufo_buffer_get_host_array (buffer, NULL); guint n_pixels = requisition->dims[0] * requisition->dims[1]; @@ -157,6 +170,8 @@ ufo_edf_reader_read (UfoReader *reader, for (guint i = 0; i < n_pixels; i++) conv[i] = g_ntohl (conv[i]); } + + return to_skip + 1; } static void @@ -192,6 +207,7 @@ ufo_edf_reader_get_depth (const gchar *value, UfoBufferDepth *depth, guint *byte static gboolean ufo_edf_reader_get_meta (UfoReader *reader, UfoRequisition *requisition, + gsize *num_images, UfoBufferDepth *bitdepth, GError **error) { @@ -270,6 +286,7 @@ ufo_edf_reader_get_meta (UfoReader *reader, g_strfreev (key_value); } + *num_images = requisition->dims[0] * requisition->dims[1] * priv->bytes_per_sample / (priv->size - data_position); g_strfreev (tokens); g_free (header); return TRUE; diff --git a/src/readers/ufo-hdf5-reader.c b/src/readers/ufo-hdf5-reader.c index 039e520..e7c94df 100644 --- a/src/readers/ufo-hdf5-reader.c +++ b/src/readers/ufo-hdf5-reader.c @@ -116,18 +116,20 @@ ufo_hdf5_reader_data_available (UfoReader *reader) return priv->current < priv->dims[0]; } -static void +static gsize ufo_hdf5_reader_read (UfoReader *reader, UfoBuffer *buffer, UfoRequisition *requisition, guint roi_y, guint roi_height, - guint roi_step) + guint roi_step, + guint image_step) { UfoHdf5ReaderPrivate *priv; gpointer data; hid_t dst_dataspace_id; hsize_t dst_dims[2]; + gsize num_read = 0; priv = UFO_HDF5_READER_GET_PRIVATE (reader); data = ufo_buffer_get_host_array (buffer, NULL); @@ -143,12 +145,16 @@ ufo_hdf5_reader_read (UfoReader *reader, H5Dread (priv->dataset_id, H5T_NATIVE_FLOAT, dst_dataspace_id, priv->src_dataspace_id, H5P_DEFAULT, data); H5Sclose (dst_dataspace_id); - priv->current++; + num_read = MIN (image_step, priv->dims[0] - priv->current); + priv->current += num_read; + + return num_read; } static gboolean ufo_hdf5_reader_get_meta (UfoReader *reader, UfoRequisition *requisition, + gsize *num_images, UfoBufferDepth *bitdepth, GError **error) { @@ -159,6 +165,7 @@ ufo_hdf5_reader_get_meta (UfoReader *reader, requisition->n_dims = 2; requisition->dims[0] = priv->dims[2]; requisition->dims[1] = priv->dims[1]; + *num_images = priv->dims[0]; *bitdepth = UFO_BUFFER_DEPTH_32F; return TRUE; } diff --git a/src/readers/ufo-raw-reader.c b/src/readers/ufo-raw-reader.c index d9b06b9..f57c0f8 100644 --- a/src/readers/ufo-raw-reader.c +++ b/src/readers/ufo-raw-reader.c @@ -123,19 +123,23 @@ ufo_raw_reader_data_available (UfoReader *reader) return priv->fp != NULL && pos >= 0 && (((gulong) pos) + priv->pre_offset + priv->frame_size) <= priv->total_size; } -static void +static gsize ufo_raw_reader_read (UfoReader *reader, UfoBuffer *buffer, UfoRequisition *requisition, guint roi_y, guint roi_height, - guint roi_step) + guint roi_step, + guint image_step) { UfoRawReaderPrivate *priv; gchar *data; + gsize to_skip; + gsize page_size; priv = UFO_RAW_READER_GET_PRIVATE (reader); data = (gchar *) ufo_buffer_get_host_array (buffer, NULL); + page_size = priv->frame_size + priv->pre_offset + priv->post_offset; fseek (priv->fp, priv->pre_offset, SEEK_CUR); @@ -144,11 +148,18 @@ ufo_raw_reader_read (UfoReader *reader, g_warning ("Could not read enough data"); fseek (priv->fp, priv->post_offset, SEEK_CUR); + + /* Skip the desired number of images */ + to_skip = MIN (image_step - 1, (priv->total_size - (gsize) ftell (priv->fp)) / page_size); + fseek (priv->fp, to_skip * page_size, SEEK_CUR); + + return to_skip + 1; } static gboolean ufo_raw_reader_get_meta (UfoReader *reader, UfoRequisition *requisition, + gsize *num_images, UfoBufferDepth *bitdepth, GError **error) { @@ -158,6 +169,7 @@ ufo_raw_reader_get_meta (UfoReader *reader, requisition->n_dims = 2; requisition->dims[0] = priv->width; requisition->dims[1] = priv->height; + *num_images = priv->total_size / priv->frame_size; *bitdepth = priv->bitdepth; return TRUE; } diff --git a/src/readers/ufo-reader.c b/src/readers/ufo-reader.c index 499abf0..f831770 100644 --- a/src/readers/ufo-reader.c +++ b/src/readers/ufo-reader.c @@ -55,21 +55,23 @@ ufo_reader_data_available (UfoReader *reader) gboolean ufo_reader_get_meta (UfoReader *reader, UfoRequisition *requisition, + gsize *num_images, UfoBufferDepth *bitdepth, GError **error) { - return UFO_READER_GET_IFACE (reader)->get_meta (reader, requisition, bitdepth, error); + return UFO_READER_GET_IFACE (reader)->get_meta (reader, requisition, num_images, bitdepth, error); } -void +gsize ufo_reader_read (UfoReader *reader, UfoBuffer *buffer, UfoRequisition *requisition, guint roi_y, guint roi_height, - guint roi_step) + guint roi_step, + guint image_step) { - UFO_READER_GET_IFACE (reader)->read (reader, buffer, requisition, roi_y, roi_height, roi_step); + return UFO_READER_GET_IFACE (reader)->read (reader, buffer, requisition, roi_y, roi_height, roi_step, image_step); } static void diff --git a/src/readers/ufo-reader.h b/src/readers/ufo-reader.h index 52c6c4b..0752506 100644 --- a/src/readers/ufo-reader.h +++ b/src/readers/ufo-reader.h @@ -48,14 +48,16 @@ struct _UfoReaderIface { gboolean (*data_available) (UfoReader *reader); gboolean (*get_meta) (UfoReader *reader, UfoRequisition *requisition, + gsize *num_images, guint *bitdepth, GError **error); - void (*read) (UfoReader *reader, + gsize (*read) (UfoReader *reader, UfoBuffer *buffer, UfoRequisition *requisition, guint roi_y, guint roi_height, - guint roi_step); + guint roi_step, + guint image_step); }; gboolean ufo_reader_can_open (UfoReader *reader, @@ -68,14 +70,16 @@ void ufo_reader_close (UfoReader *reader); gboolean ufo_reader_data_available (UfoReader *reader); gboolean ufo_reader_get_meta (UfoReader *reader, UfoRequisition *requisition, + gsize *num_images, UfoBufferDepth *bitdepth, GError **error); -void ufo_reader_read (UfoReader *reader, +gsize ufo_reader_read (UfoReader *reader, UfoBuffer *buffer, UfoRequisition *requisition, guint roi_y, guint roi_height, - guint roi_step); + guint roi_step, + guint image_step); GType ufo_reader_get_type (void); diff --git a/src/readers/ufo-tiff-reader.c b/src/readers/ufo-tiff-reader.c index 0bac745..678a97b 100644 --- a/src/readers/ufo-tiff-reader.c +++ b/src/readers/ufo-tiff-reader.c @@ -26,6 +26,7 @@ struct _UfoTiffReaderPrivate { TIFF *tiff; gboolean more; + gsize num_images; }; static void ufo_reader_interface_init (UfoReaderIface *iface); @@ -59,8 +60,9 @@ ufo_tiff_reader_open (UfoReader *reader, UfoTiffReaderPrivate *priv; priv = UFO_TIFF_READER_GET_PRIVATE (reader); + priv->num_images = 0; priv->tiff = TIFFOpen (filename, "r"); - priv->more = TRUE; + priv->more = FALSE; if (priv->tiff == NULL) { g_set_error (error, UFO_TASK_ERROR, UFO_TASK_ERROR_SETUP, @@ -68,8 +70,18 @@ ufo_tiff_reader_open (UfoReader *reader, return FALSE; } - for (guint i = 0; i < start; i++) - priv->more = TIFFReadDirectory (priv->tiff) == 1; + do { + priv->num_images++; + } while (TIFFReadDirectory(priv->tiff)); + + if (start < priv->num_images) { + priv->more = TRUE; + if (TIFFSetDirectory (priv->tiff, start) != 1) { + g_set_error (error, UFO_TASK_ERROR, UFO_TASK_ERROR_SETUP, + "Cannot find first image in %s", filename); + return FALSE; + } + } return TRUE; } @@ -172,16 +184,18 @@ read_64_bit_data (UfoTiffReaderPrivate *priv, g_free (src); } -static void +static gsize ufo_tiff_reader_read (UfoReader *reader, UfoBuffer *buffer, UfoRequisition *requisition, guint roi_y, guint roi_height, - guint roi_step) + guint roi_step, + guint image_step) { UfoTiffReaderPrivate *priv; guint16 bits; + gsize num_read = 0; priv = UFO_TIFF_READER_GET_PRIVATE (reader); @@ -192,12 +206,21 @@ ufo_tiff_reader_read (UfoReader *reader, else read_data (priv, buffer, requisition, bits, roi_y, roi_height, roi_step); - priv->more = TIFFReadDirectory (priv->tiff) == 1; + do { + priv->more = TIFFReadDirectory (priv->tiff) == 1; + num_read++; + if (!priv->more) { + break; + } + } while (num_read < image_step); + + return num_read; } static gboolean ufo_tiff_reader_get_meta (UfoReader *reader, UfoRequisition *requisition, + gsize *num_images, UfoBufferDepth *bitdepth, GError **error) { @@ -219,6 +242,7 @@ ufo_tiff_reader_get_meta (UfoReader *reader, requisition->dims[0] = (gsize) width; requisition->dims[1] = (gsize) height; requisition->dims[2] = samples == 3 ? 3 : 0; + *num_images = priv->num_images; switch (bits_per_sample) { case 8: diff --git a/src/ufo-read-task.c b/src/ufo-read-task.c index 1587031..b2a0457 100644 --- a/src/ufo-read-task.c +++ b/src/ufo-read-task.c @@ -70,6 +70,7 @@ struct _UfoReadTaskPrivate { guint current; guint step; guint start; + guint image_start; guint number; guint retries; guint retry_timeout; @@ -82,6 +83,7 @@ struct _UfoReadTaskPrivate { guint roi_y; guint roi_height; guint roi_step; + guint image_step; UfoReader *reader; UfoEdfReader *edf_reader; @@ -110,11 +112,13 @@ enum { PROP_0, PROP_PATH, PROP_START, + PROP_IMAGE_START, PROP_NUMBER, PROP_STEP, PROP_ROI_Y, PROP_ROI_HEIGHT, PROP_ROI_STEP, + PROP_IMAGE_STEP, PROP_CONVERT, PROP_RAW_WIDTH, PROP_RAW_HEIGHT, @@ -224,7 +228,6 @@ ufo_read_task_setup (UfoTask *task, return; } - priv->start = 0; priv->current = 0; } @@ -258,69 +261,73 @@ ufo_read_task_get_requisition (UfoTask *task, { UfoReadTaskPrivate *priv; const gchar *filename; + gsize num_images = 0; priv = UFO_READ_TASK_GET_PRIVATE (UFO_READ_TASK (task)); - if (priv->reader == NULL) { - filename = (gchar *) priv->current_element->data; - priv->reader = get_reader (priv, filename); - - if (!ufo_reader_open (priv->reader, filename, priv->start, error)) - return; - - priv->start = 0; - } - - if (!ufo_reader_data_available (priv->reader)) { + if (!priv->reader || !ufo_reader_data_available (priv->reader)) { GList *last_element; guint tries; - ufo_reader_close (priv->reader); - last_element = priv->current_element; - priv->current_element = g_list_nth (priv->current_element, priv->step); - - if (priv->current_element == NULL) { - if (priv->retries == 0 || priv->current == priv->number) { - priv->done = TRUE; - priv->reader = NULL; - return; + while (TRUE) { + /* Keep skipping files until we find one with enough images to start + * reading at priv->image_start index. */ + if (priv->reader) { + ufo_reader_close (priv->reader); + last_element = priv->current_element; + priv->current_element = g_list_nth (priv->current_element, priv->step); } - for (tries = 0; tries < priv->retries && priv->current_element == NULL; tries++) { - GList *new_list; - GList *match; - - g_debug ("read: retry %i/%i, waiting %is for new files", tries + 1, priv->retries, priv->retry_timeout); - g_usleep (priv->retry_timeout * G_USEC_PER_SEC); - new_list = g_list_sort (read_filenames (priv), (GCompareFunc) g_strcmp0); - match = g_list_find_custom (new_list, last_element->data, (GCompareFunc) g_strcmp0); - - if (match != g_list_last (new_list)) { - g_list_free_full (priv->filenames, (GDestroyNotify) g_free); - priv->filenames = new_list; - priv->current_element = g_list_next (match); - } - else { - g_list_free_full (new_list, (GDestroyNotify) g_free); - } + if (priv->current_element == NULL) { + if (priv->retries == 0 || priv->current == priv->number) { + priv->done = TRUE; + priv->reader = NULL; + return; + } + + for (tries = 0; tries < priv->retries && priv->current_element == NULL; tries++) { + GList *new_list; + GList *match; + + g_debug ("read: retry %i/%i, waiting %is for new files", tries + 1, priv->retries, priv->retry_timeout); + g_usleep (priv->retry_timeout * G_USEC_PER_SEC); + new_list = g_list_sort (read_filenames (priv), (GCompareFunc) g_strcmp0); + match = g_list_find_custom (new_list, last_element->data, (GCompareFunc) g_strcmp0); + + if (match != g_list_last (new_list)) { + g_list_free_full (priv->filenames, (GDestroyNotify) g_free); + priv->filenames = new_list; + priv->current_element = g_list_next (match); + } + else { + g_list_free_full (new_list, (GDestroyNotify) g_free); + } + } + + if (priv->current_element == NULL) { + priv->done = TRUE; + priv->reader = NULL; + return; + } } - if (priv->current_element == NULL) { - priv->done = TRUE; - priv->reader = NULL; + filename = (gchar *) priv->current_element->data; + priv->reader = get_reader (priv, filename); + + if (!ufo_reader_open (priv->reader, filename, priv->image_start, error)) return; + if (!ufo_reader_get_meta (priv->reader, requisition, &num_images, &priv->depth, error)) + return; + + if (priv->image_start >= num_images) { + priv->image_start -= num_images; + } else { + priv->image_start = 0; + break; } } - - filename = (gchar *) priv->current_element->data; - priv->reader = get_reader (priv, filename); - - if (!ufo_reader_open (priv->reader, filename, 0, error)) - return; } - if (!ufo_reader_get_meta (priv->reader, requisition, &priv->depth, error)) - return; if (priv->depth > 32) /* @@ -376,13 +383,16 @@ ufo_read_task_generate (UfoTask *task, UfoRequisition *requisition) { UfoReadTaskPrivate *priv; + guint num_processed; priv = UFO_READ_TASK_GET_PRIVATE (UFO_READ_TASK (task)); if (priv->current == priv->number || priv->done) return FALSE; - ufo_reader_read (priv->reader, output, requisition, priv->roi_y, priv->roi_height, priv->roi_step); + num_processed = ufo_reader_read (priv->reader, output, requisition, priv->roi_y, priv->roi_height, + priv->roi_step, priv->image_step); + priv->image_start = priv->image_step - num_processed; if ((priv->depth != UFO_BUFFER_DEPTH_32F) && priv->convert) ufo_buffer_convert (output, priv->depth); @@ -416,12 +426,18 @@ ufo_read_task_set_property (GObject *object, case PROP_ROI_STEP: priv->roi_step = g_value_get_uint (value); break; + case PROP_IMAGE_STEP: + priv->image_step = g_value_get_uint (value); + break; case PROP_CONVERT: priv->convert = g_value_get_boolean (value); break; case PROP_START: priv->start = g_value_get_uint (value); break; + case PROP_IMAGE_START: + priv->image_start = g_value_get_uint (value); + break; case PROP_NUMBER: priv->number = g_value_get_uint (value); break; @@ -479,12 +495,18 @@ ufo_read_task_get_property (GObject *object, case PROP_ROI_STEP: g_value_set_uint (value, priv->roi_step); break; + case PROP_IMAGE_STEP: + g_value_set_uint (value, priv->image_step); + break; case PROP_CONVERT: g_value_set_boolean (value, priv->convert); break; case PROP_START: g_value_set_uint (value, priv->start); break; + case PROP_IMAGE_START: + g_value_set_uint (value, priv->image_start); + break; case PROP_NUMBER: g_value_set_uint (value, priv->number); break; @@ -613,6 +635,13 @@ ufo_read_task_class_init(UfoReadTaskClass *klass) 1, G_MAXUINT, 1, G_PARAM_READWRITE); + properties[PROP_IMAGE_STEP] = + g_param_spec_uint ("image-step", + "Read every \"step\" image", + "Read every \"step\" image", + 1, G_MAXUINT, 1, + G_PARAM_READWRITE); + properties[PROP_CONVERT] = g_param_spec_boolean ("convert", "Enable automatic conversion", @@ -627,10 +656,17 @@ ufo_read_task_class_init(UfoReadTaskClass *klass) 0, G_MAXUINT, 0, G_PARAM_READWRITE); + properties[PROP_IMAGE_START] = + g_param_spec_uint ("image-start", + "Offset to the first read image in a multi-image file", + "Offset to the first read image in a multi-image file", + 0, G_MAXUINT, 0, + G_PARAM_READWRITE); + properties[PROP_NUMBER] = g_param_spec_uint ("number", - "Number of files that will be read at most", - "Number of files that will be read at most", + "Number of images that will be read at most", + "Number of images that will be read at most", 0, G_MAXUINT, G_MAXUINT, G_PARAM_READWRITE); @@ -708,8 +744,10 @@ ufo_read_task_init(UfoReadTask *self) priv->roi_y = 0; priv->roi_height = 0; priv->roi_step = 1; + priv->image_step = 1; priv->convert = TRUE; priv->start = 0; + priv->image_start = 0; priv->number = G_MAXUINT; priv->retries = 0; priv->retry_timeout = 1; -- cgit v1.2.1 From 9d37808d929b69d1526362ddc17f60774c08ac54 Mon Sep 17 00:00:00 2001 From: Tomas Farago Date: Wed, 1 Dec 2021 16:31:45 +0100 Subject: Test multi-image readers --- tests/CMakeLists.txt | 9 ++++++ tests/check-multipage-readers | 24 ++++++++++++++++ tests/make-input-multipage-readers | 18 ++++++++++++ tests/meson.build | 11 ++++++- tests/test-multipage-readers.sh | 59 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 1 deletion(-) create mode 100755 tests/check-multipage-readers create mode 100755 tests/make-input-multipage-readers create mode 100755 tests/test-multipage-readers.sh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8de94f1..d2edac1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -23,3 +23,12 @@ add_test(test_core_149 add_test(test_nlm ${BASH} "${CMAKE_CURRENT_SOURCE_DIR}/test-nlm.sh") + +add_test(test_multipage_readers + ${BASH} "${CMAKE_CURRENT_SOURCE_DIR}/test-multipage-readers.sh") + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/make-input-multipage-readers + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tests) + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/check-multipage-readers + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tests) diff --git a/tests/check-multipage-readers b/tests/check-multipage-readers new file mode 100755 index 0000000..4e1bd9e --- /dev/null +++ b/tests/check-multipage-readers @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import sys +import numpy as np +import tifffile + + +def main(gt): + im = tifffile.imread('multipage-out.tif') + if im.ndim == 2: + im = im[np.newaxis] + + collected = [] + for i in range(im.shape[0]): + collected.append(int(im[i, 0, 0])) + if int(gt[i]) != collected[i]: + print('Sequences do not match', gt[:i + 1], collected[:i + 1]) + return 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tests/make-input-multipage-readers b/tests/make-input-multipage-readers new file mode 100755 index 0000000..a565122 --- /dev/null +++ b/tests/make-input-multipage-readers @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import h5py +import numpy as np +import tifffile + + +base = np.ones((15, 128, 128), dtype=np.uint16) * np.arange(15, dtype=np.uint16)[:, np.newaxis, np.newaxis] +fmt = 'multipage-image-{:>02}.{}' + + +for i in range(4): + current = base + i * len(base) + tifffile.imsave(fmt.format(i, 'tif'), current) + with h5py.File(fmt.format(i, 'h5'), 'w') as f: + dset = f.create_dataset('images', data=current) + with open(fmt.format(i, 'raw'), 'wb') as f: + f.write(current.tostring()) diff --git a/tests/meson.build b/tests/meson.build index 6b720d2..4ab4156 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -4,7 +4,8 @@ tests = [ 'test-161', 'test-core-149', 'test-file-write-regression', - 'test-nlm' + 'test-nlm', + 'test-multipage-readers' ] tiffinfo = find_program('tiffinfo', required : false) @@ -17,6 +18,14 @@ test_env = [ 'UFO_PLUGIN_PATH=@0@'.format(join_paths(meson.build_root(), 'src')) ] +configure_file(input: 'make-input-multipage-readers', + output: 'make-input-multipage-readers', + copy: true) + +configure_file(input: 'check-multipage-readers', + output: 'check-multipage-readers', + copy: true) + foreach t: tests test(t, find_program('@0@.sh'.format(t)), env: test_env) endforeach diff --git a/tests/test-multipage-readers.sh b/tests/test-multipage-readers.sh new file mode 100755 index 0000000..ab5845b --- /dev/null +++ b/tests/test-multipage-readers.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +tests/make-input-multipage-readers + +# TIFF +ufo-launch -q read path=multipage-image*.tif image-start=5 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 5 13 21 29 37 45 53 +if [ $? -ne 0 ]; then + exit 1; +fi +ufo-launch -q read path=multipage-image*.tif image-start=17 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 17 25 33 41 49 57 +if [ $? -ne 0 ]; then + exit 1; +fi +ufo-launch -q read path=multipage-image*.tif image-start=5 image-step=30 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 5 35 +if [ $? -ne 0 ]; then + exit 1; +fi +ufo-launch -q read path=multipage-image*.tif image-start=17 image-step=30 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 17 47 +if [ $? -ne 0 ]; then + exit 1; +fi + +# RAW +ufo-launch -q read raw-width=128 raw-height=128 raw-bitdepth=16 path=multipage-image*.raw image-start=5 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 5 13 21 29 37 45 53 +if [ $? -ne 0 ]; then + exit 1; +fi +ufo-launch -q read raw-width=128 raw-height=128 raw-bitdepth=16 path=multipage-image*.raw image-start=17 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 17 25 33 41 49 57 +if [ $? -ne 0 ]; then + exit 1; +fi +ufo-launch -q read raw-width=128 raw-height=128 raw-bitdepth=16 path=multipage-image*.raw image-start=5 image-step=30 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 5 35 +if [ $? -ne 0 ]; then + exit 1; +fi +ufo-launch -q read raw-width=128 raw-height=128 raw-bitdepth=16 path=multipage-image*.raw image-start=17 image-step=30 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 17 47 +if [ $? -ne 0 ]; then + exit 1; +fi + +# HDF5 (no multiple files reading support, so just work with one) +ufo-launch -q read path=multipage-image-00.h5:/images image-start=5 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 5 13 +if [ $? -ne 0 ]; then + exit 1; +fi +ufo-launch -q read path=multipage-image-00.h5:/images image-start=5 image-step=80 ! write filename=multipage-out.tif tiff-bigtiff=False +tests/check-multipage-readers 5 +if [ $? -ne 0 ]; then + exit 1; +fi -- cgit v1.2.1 From a75fe709b9d3066af22c3d54c8fb1066afd7e4cc Mon Sep 17 00:00:00 2001 From: Tomas Farago Date: Fri, 3 Dec 2021 14:34:23 +0100 Subject: read: remove retries --- src/ufo-read-task.c | 75 +++-------------------------------------------------- 1 file changed, 3 insertions(+), 72 deletions(-) diff --git a/src/ufo-read-task.c b/src/ufo-read-task.c index b2a0457..f4818e0 100644 --- a/src/ufo-read-task.c +++ b/src/ufo-read-task.c @@ -72,8 +72,6 @@ struct _UfoReadTaskPrivate { guint start; guint image_start; guint number; - guint retries; - guint retry_timeout; gboolean done; gboolean single; @@ -126,8 +124,6 @@ enum { PROP_RAW_PRE_OFFSET, PROP_RAW_POST_OFFSET, PROP_TYPE, - PROP_RETRIES, - PROP_RETRY_TIMEOUT, N_PROPERTIES }; @@ -222,12 +218,6 @@ ufo_read_task_setup (UfoTask *task, return; } - if (priv->number == G_MAXUINT && priv->retries > 0) { - g_set_error (error, UFO_TASK_ERROR, UFO_TASK_ERROR_SETUP, - "`retries' but not `number' set"); - return; - } - priv->current = 0; } @@ -266,49 +256,18 @@ ufo_read_task_get_requisition (UfoTask *task, priv = UFO_READ_TASK_GET_PRIVATE (UFO_READ_TASK (task)); if (!priv->reader || !ufo_reader_data_available (priv->reader)) { - GList *last_element; - guint tries; - while (TRUE) { /* Keep skipping files until we find one with enough images to start * reading at priv->image_start index. */ if (priv->reader) { ufo_reader_close (priv->reader); - last_element = priv->current_element; priv->current_element = g_list_nth (priv->current_element, priv->step); } if (priv->current_element == NULL) { - if (priv->retries == 0 || priv->current == priv->number) { - priv->done = TRUE; - priv->reader = NULL; - return; - } - - for (tries = 0; tries < priv->retries && priv->current_element == NULL; tries++) { - GList *new_list; - GList *match; - - g_debug ("read: retry %i/%i, waiting %is for new files", tries + 1, priv->retries, priv->retry_timeout); - g_usleep (priv->retry_timeout * G_USEC_PER_SEC); - new_list = g_list_sort (read_filenames (priv), (GCompareFunc) g_strcmp0); - match = g_list_find_custom (new_list, last_element->data, (GCompareFunc) g_strcmp0); - - if (match != g_list_last (new_list)) { - g_list_free_full (priv->filenames, (GDestroyNotify) g_free); - priv->filenames = new_list; - priv->current_element = g_list_next (match); - } - else { - g_list_free_full (new_list, (GDestroyNotify) g_free); - } - } - - if (priv->current_element == NULL) { - priv->done = TRUE; - priv->reader = NULL; - return; - } + priv->done = TRUE; + priv->reader = NULL; + return; } filename = (gchar *) priv->current_element->data; @@ -459,12 +418,6 @@ ufo_read_task_set_property (GObject *object, case PROP_TYPE: priv->type = g_value_get_enum (value); break; - case PROP_RETRIES: - priv->retries = g_value_get_uint (value); - break; - case PROP_RETRY_TIMEOUT: - priv->retry_timeout = g_value_get_uint (value); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -528,12 +481,6 @@ ufo_read_task_get_property (GObject *object, case PROP_TYPE: g_value_set_enum (value, priv->type); break; - case PROP_RETRIES: - g_value_set_uint (value, priv->retries); - break; - case PROP_RETRY_TIMEOUT: - g_value_set_uint (value, priv->retry_timeout); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -713,20 +660,6 @@ ufo_read_task_class_init(UfoReadTaskClass *klass) TYPE_UNSPECIFIED, G_PARAM_READWRITE); - properties[PROP_RETRIES] = - g_param_spec_uint ("retries", - "Number of read retries", - "Number of read retries", - 0, G_MAXUINT, 0, - G_PARAM_READWRITE); - - properties[PROP_RETRY_TIMEOUT] = - g_param_spec_uint ("retry-timeout", - "Time in seconds to wait between retries", - "Time in seconds to wait between retries", - 0, G_MAXUINT, 1, - G_PARAM_READWRITE); - for (guint i = PROP_0 + 1; i < N_PROPERTIES; i++) g_object_class_install_property (gobject_class, i, properties[i]); @@ -749,8 +682,6 @@ ufo_read_task_init(UfoReadTask *self) priv->start = 0; priv->image_start = 0; priv->number = G_MAXUINT; - priv->retries = 0; - priv->retry_timeout = 1; priv->depth = UFO_BUFFER_DEPTH_32F; priv->edf_reader = ufo_edf_reader_new (); -- cgit v1.2.1 From 9f49adb9b922cecc6cdabb437701a2005953b265 Mon Sep 17 00:00:00 2001 From: Tomas Farago Date: Fri, 3 Dec 2021 14:36:46 +0100 Subject: docs: remove retries and retry-timeout --- docs/generators.rst | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docs/generators.rst b/docs/generators.rst index 9f063ab..d0925f5 100644 --- a/docs/generators.rst +++ b/docs/generators.rst @@ -88,18 +88,6 @@ File reader example, to load `.foo` files as raw files, set the ``type`` property to `raw`. - .. gobj:prop:: retries:uint - - Set the number of retries in case files do not exist yet and are being - written. If you set this, you *must* also set ``number`` otherwise you - would have to wait basically forever for the execution to finish. Note, - that only files are considered which come after the last successful - filename. - - .. gobj:prop:: retry-timeout:uint - - Seconds to wait before reading new files. - Memory reader ============= -- cgit v1.2.1