Build ROOF sinograms

14 files changed, 371 insertions, 132 deletions

diff --git a/src/ufo-roof-buffer.c b/src/ufo-roof-buffer.c
index 179d153..bac940c 100644
--- a/src/ufo-roof-buffer.c
+++ b/src/ufo-roof-buffer.c
@@ -9,15 +9,24 @@
 
 // This is currently not thread safe. With dual-filter architecture this will be called sequentially.
 
-UfoRoofBuffer *ufo_roof_buffer_new(UfoRoofConfig *cfg, guint max_datasets, GError **error) {
+UfoRoofBuffer *ufo_roof_buffer_new(UfoRoofConfig *cfg, guint n_dims, guint max_datasets, GError **error) {
+    if ((n_dims < 1)||(n_dims > 2))
+	roof_new_error(error, "Unsupported number of dimmensions %u (only plain and 2D ROOF structure is currently supported)", n_dims);
+
     UfoRoofBuffer *buffer = (UfoRoofBuffer*)calloc(1, sizeof(UfoRoofBuffer));
     if (!buffer) roof_new_error(error, "Can't allocate UfoRoofBuffer");
 
     buffer->max_datasets = max_datasets;
     buffer->ring_size = cfg->buffer_size;
     buffer->drop_buffers = cfg->drop_buffers;
-    buffer->fragment_size = cfg->payload_size;
+    buffer->n_dims = n_dims;
     buffer->dataset_size = cfg->dataset_size;
+    buffer->dataset_dims[0] = cfg->fan_bins * cfg->bit_depth / 8;
+    buffer->dataset_dims[1] = cfg->fan_projections;
+    buffer->fragment_size = cfg->payload_size;
+    buffer->fragment_dims[0] = cfg->channels_per_module * cfg->bit_depth / 8;
+    buffer->fragment_dims[1] = buffer->fragment_size / buffer->fragment_dims[0];
+
     buffer->fragments_per_dataset = buffer->dataset_size / buffer->fragment_size;
     buffer->fragments_per_stream = buffer->fragments_per_dataset / cfg->n_streams;
 //    printf("Configuration: dataset: %u - %u fragments (%u streams x %u) x %u bytes\n", buffer->dataset_size, buffer->fragments_per_dataset, cfg->n_streams, buffer->fragments_per_stream, buffer->fragment_size);
@@ -96,15 +105,25 @@ gboolean ufo_roof_buffer_set_fragment(UfoRoofBuffer *buffer, guint stream_id, gu
             // The updates may happen after writting/reading is finished.
     }
 
-        // FIXME: This is builds events as it read from file in roof v.1 code. We can assemble fan projections directly here.
-    uint8_t *dataset_buffer = buffer->ring_buffer + buffer_id * buffer->dataset_size;
-    uint8_t *fragment_buffer = dataset_buffer + (stream_id * buffer->fragments_per_stream + fragment_id) * buffer->fragment_size;
-    
 /*    printf("buffer: %u (%u), packet: %u (%ux%u %u), packet_size: %u [%x]\n", 
         buffer_id, dataset_id, stream_id * buffer->fragments_per_stream + fragment_id, stream_id, buffer->fragments_per_stream, fragment_id, buffer->fragment_size,
         ((uint32_t*)fragment)[0]
     );*/
-    memcpy(fragment_buffer, fragment, buffer->fragment_size);
+
+    uint8_t *dataset_buffer = buffer->ring_buffer + buffer_id * buffer->dataset_size;
+    if (buffer->n_dims == 2) {
+	uint8_t *fragment_buffer = dataset_buffer + 
+		stream_id * buffer->fragment_dims[0] +					// x-coordinate
+		(fragment_id * buffer->fragment_dims[1]) * buffer->dataset_dims[0];	// y-coordinate
+
+	for (int i = 0; i < buffer->fragment_dims[1]; ++i) {
+	    memcpy(fragment_buffer + i * buffer->dataset_dims[0], fragment + i * buffer->fragment_dims[0], buffer->fragment_dims[0]);
+	}
+    } else {
+	    // 1D stracture, simply putting fragment at the appropriate position in the stream
+	uint8_t *fragment_buffer = dataset_buffer + (stream_id * buffer->fragments_per_stream + fragment_id) * buffer->fragment_size;
+	memcpy(fragment_buffer, fragment, buffer->fragment_size);
+    }
 
         // FIXME: Sanity checks: verify is not a dublicate fragment?
     atomic_fetch_add(&buffer->n_fragments[buffer_id], 1);
@@ -128,6 +147,7 @@ gboolean ufo_roof_buffer_get_dataset(UfoRoofBuffer *buffer, gpointer output_buff
     if (buffer->n_fragments[buffer_id] < buffer->fragments_per_dataset) return FALSE;
 
     memcpy(output_buffer, dataset_buffer, buffer->dataset_size);
+
     buffer->n_fragments[buffer_id] = 0;
     buffer->current_id += 1;
 
diff --git a/src/ufo-roof-buffer.h b/src/ufo-roof-buffer.h
index c4c8474..7ebaec9 100644
--- a/src/ufo-roof-buffer.h
+++ b/src/ufo-roof-buffer.h
@@ -15,16 +15,19 @@ struct _UfoRoofBuffer {
 
 
     guint			max_datasets;				// Only the specified number of datasets will be buffered, the rest will be silently dropped
+    guint			n_dims;					// Indicates if we just assemble one fragment after another or there is 2D/3D data structure (ROOF)
     guint                       dataset_size;                           // Size (in bytes) of a full dataset
+    guint			dataset_dims[2];			// x (in bytes), y (in rows)
     guint                       fragment_size;                          // Size (in bytes) of a single fragment (we expect fixed-size fragments at the moment)
-    
+    guint			fragment_dims[2];			// x (in bytes), y (in rows)
+
     guint                       fragments_per_dataset;                  // Number of packets in dataset (used to compute when dataset is ready)
     guint                       fragments_per_stream;                   // Number of packets in each of data streams (used to compute when dataset is ready)
 };
 
 typedef struct _UfoRoofBuffer	UfoRoofBuffer;
 
-UfoRoofBuffer *ufo_roof_buffer_new(UfoRoofConfig *cfg, guint max_datasets, GError **error);
+UfoRoofBuffer *ufo_roof_buffer_new(UfoRoofConfig *cfg, guint n_dims, guint max_datasets, GError **error);
 void ufo_roof_buffer_free(UfoRoofBuffer *buf);
 
 gboolean ufo_roof_buffer_set_fragment(UfoRoofBuffer *buffer, guint stream_id, guint fragment_id, gconstpointer fragment, GError **error);
diff --git a/src/ufo-roof-build-task.c b/src/ufo-roof-build-task.c
index e5e5518..8af44db 100644
--- a/src/ufo-roof-build-task.c
+++ b/src/ufo-roof-build-task.c
@@ -29,12 +29,19 @@
 #include "ufo-roof-buffer.h"
 #include "ufo-roof-build-task.h"
 
+typedef enum {
+    BUILD_AUTO = 0,
+    BUILD_RAW,
+    BUILD_SINO,
+    BUILD_UFO
+} BuildType;
 
 struct _UfoRoofBuildTaskPrivate {
     gchar			*config;				// ROOF configuration file name
     UfoRoofConfig               *cfg;                                   // Parsed ROOF parameters
     UfoRoofBuffer               *buf;                                   // Ring buffer for incomming UDP packet
 
+    BuildType			build;					// What dataset do we build: ROOF sinogram or raw network data
     guint                       number;                                 // Number of datasets to read
     gboolean                    stop;                                   // Stop flag
 
@@ -51,10 +58,21 @@ G_DEFINE_TYPE_WITH_CODE (UfoRoofBuildTask, ufo_roof_build_task, UFO_TYPE_TASK_NO
 
 #define UFO_ROOF_BUILD_TASK_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE((obj), UFO_TYPE_ROOF_BUILD_TASK, UfoRoofBuildTaskPrivate))
 
+
+
+static GEnumValue build_values[] = {
+    { BUILD_AUTO,		"BUILD_AUTO",		"auto" },
+    { BUILD_RAW,		"BUILD_RAW",		"raw" },
+    { BUILD_SINO,		"BUILD_SINO",		"sino" },
+    { BUILD_UFO,		"BUILD_UFO",		"ufo" },
+    { 0, NULL, NULL}
+};
+
 enum {
     PROP_0,
     PROP_STOP,
     PROP_NUMBER,
+    PROP_BUILD,
     PROP_CONFIG,
     N_PROPERTIES
 };
@@ -83,8 +101,13 @@ ufo_roof_build_task_setup (UfoTask *task,
     if (!priv->cfg)
         roof_propagate_error(error, gerr, "roof-build-setup: ");
 
+    if (priv->build == BUILD_AUTO) {
+	if (priv->cfg->roof_mode) priv->build = BUILD_SINO;
+	else priv->build = BUILD_RAW;
+	g_object_notify_by_pspec (G_OBJECT(task), properties[PROP_BUILD]);
+    }
 
-    priv->buf = ufo_roof_buffer_new(priv->cfg, priv->number, &gerr);
+    priv->buf = ufo_roof_buffer_new(priv->cfg, (priv->build == BUILD_RAW)?1:2, priv->number, &gerr);
     if (!priv->buf) 
         roof_propagate_error(error, gerr, "roof-build-setup: ");
 
@@ -103,7 +126,7 @@ ufo_roof_build_task_finalize (GObject *object)
 
     if (priv->cfg) {
         ufo_roof_config_free(priv->cfg);
-        priv->cfg = NULL;        
+        priv->cfg = NULL;
     }
 
     if (priv->config) {
@@ -125,12 +148,21 @@ ufo_roof_build_task_get_requisition (UfoTask *task,
 {
     UfoRoofBuildTaskPrivate *priv = UFO_ROOF_BUILD_TASK_GET_PRIVATE (task);
 
-    guint bytes = priv->cfg->dataset_size;
-
-        // FIXME: Can this be made more elegant?
-    requisition->n_dims = 1;
-    requisition->dims[0] = bytes / sizeof(float) + ((bytes%sizeof(float))?1:0);
-    
+        // FIXME: Can we handle data types more elegant?
+    if (priv->build == BUILD_RAW) {
+	guint bytes = priv->cfg->dataset_size;
+	requisition->n_dims = 1;
+	requisition->dims[0] = bytes / sizeof(float) + ((bytes%sizeof(float))?1:0);
+    } else if (priv->build == BUILD_SINO) {
+	guint bytes = priv->cfg->fan_bins * priv->cfg->bit_depth / 8;
+	requisition->n_dims = 2;
+	requisition->dims[0] = bytes / sizeof(float) + ((bytes%sizeof(float))?1:0);
+	requisition->dims[1] = priv->cfg->fan_projections;
+    } else if (priv->build == BUILD_UFO) {
+	requisition->n_dims = 2;
+	requisition->dims[0] = priv->cfg->fan_bins;
+	requisition->dims[1] = priv->cfg->fan_projections;
+    }
 }
 
 static guint
@@ -222,9 +254,9 @@ ufo_roof_build_task_generate (UfoTask *task,
 {
     gboolean ready = FALSE;
     GError *gerr = NULL;
-    
+
     UfoRoofBuildTaskPrivate *priv = UFO_ROOF_BUILD_TASK_GET_PRIVATE (task);
-//    UfoRoofConfig *cfg = priv->cfg;
+    UfoRoofConfig *cfg = priv->cfg;
     UfoRoofBuffer *buf = priv->buf;
 
     void *output_buffer = ufo_buffer_get_host_array(output, NULL);
@@ -235,6 +267,20 @@ ufo_roof_build_task_generate (UfoTask *task,
     ready = ufo_roof_buffer_get_dataset(buf, output_buffer, &gerr);
     if (gerr) roof_print_error(gerr);
 
+    if (priv->build == BUILD_UFO) {
+	switch (cfg->bit_depth) {
+	    case 8:
+		ufo_buffer_convert(output, UFO_BUFFER_DEPTH_8U);
+		break;
+	    case 16:
+		ufo_buffer_convert(output, UFO_BUFFER_DEPTH_16U);
+		break;
+	    case 32:
+		ufo_buffer_convert(output, UFO_BUFFER_DEPTH_32U);
+		break;
+	}
+    }
+
 	// FIXME: Or shall we start from counting from the ID of the first registerd dataset
     if ((priv->number)&&(buf->current_id >= priv->number)) {
 //	printf("%u datasets processed, stopping\n", buf->current_id);
@@ -242,8 +288,7 @@ ufo_roof_build_task_generate (UfoTask *task,
         g_object_notify_by_pspec (G_OBJECT(task), properties[PROP_STOP]);
     }
 
-
-    if ((priv->number < 100)||((buf->current_id - priv->announced) > 1000)) {
+    if (((priv->number > 0)&&(priv->number <= 100))||((buf->current_id - priv->announced) > 1000)) {
 	printf("Generating dataset %i (%s), next: %u out of %u)\n", buf->current_id, ready?"yes":" no", buf->n_fragments[buf->current_id%buf->ring_size], buf->fragments_per_dataset);
 	priv->announced = buf->current_id;
     }
@@ -270,6 +315,13 @@ ufo_roof_build_task_set_property (GObject *object,
         case PROP_NUMBER:
             priv->number = g_value_get_uint (value);
             break;
+        case PROP_BUILD:
+            priv->build = g_value_get_enum (value);
+	    if ((priv->build == BUILD_AUTO)&&(priv->cfg)) {
+		if (priv->cfg->roof_mode) priv->build = BUILD_SINO;
+		else priv->build = BUILD_RAW;
+	    }
+            break;
         default:
             G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
             break;
@@ -294,6 +346,9 @@ ufo_roof_build_task_get_property (GObject *object,
         case PROP_NUMBER:
             g_value_set_uint (value, priv->number);
             break;
+        case PROP_BUILD:
+            g_value_set_enum (value, priv->build);
+            break;
         default:
             G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
             break;
@@ -342,6 +397,13 @@ ufo_roof_build_task_class_init (UfoRoofBuildTaskClass *klass)
             0, G_MAXUINT, 0,
             G_PARAM_READWRITE);
 
+    properties[PROP_BUILD] =
+        g_param_spec_enum ("build",
+            "Build type (\"raw\", \"sino\", \"ufo\")",
+            "Build type (\"raw\" - raw data, \"sino\" - arrange in sinogram, \"ufo\" - arrange in sinogram and convert UFO floating-point format)",
+            g_enum_register_static ("build", build_values),
+            0, G_PARAM_READWRITE);
+
 
     for (guint i = PROP_0 + 1; i < N_PROPERTIES; i++)
         g_object_class_install_property (oclass, i, properties[i]);
diff --git a/src/ufo-roof-config.c b/src/ufo-roof-config.c
index 812d4a2..4788a2a 100644
--- a/src/ufo-roof-config.c
+++ b/src/ufo-roof-config.c
@@ -5,6 +5,7 @@
 
 #include <ufo/ufo.h>
 
+#include "ufo-roof.h"
 #include "ufo-roof-error.h"
 #include "ufo-roof-config.h"
 
@@ -59,10 +60,13 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
 //    JsonNode *node;
     JsonObject *root = NULL;
     JsonObject *hardware = NULL;
+    JsonObject *geometry = NULL;
+    JsonObject *optics = NULL;
     JsonObject *network = NULL;
     JsonObject *performance = NULL;
     JsonObject *simulation = NULL;
-    
+    JsonObject *reconstruction = NULL;
+
     GError *gerr = NULL;
     
     priv = (UfoRoofConfigPrivate*)malloc(sizeof(UfoRoofConfigPrivate));
@@ -73,11 +77,20 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
 	// Set defaults
     cfg = &priv->cfg;
 
-    cfg->port = 4000;
+    cfg->roof_mode = FALSE;
+    cfg->n_planes = 2;
+    cfg->n_modules = 16;
+    cfg->channels_per_module = 16;
+    cfg->bit_depth = 16;
+    cfg->samples_per_rotation = 1000;
+    cfg->sample_rate = 0;
+    cfg->imaging_rate = 0;
+
+    cfg->port = 52067;
     cfg->n_streams = 1;
     cfg->protocol = "udp";
     cfg->network_timeout = 10000000;
-    cfg->header_size = 0;
+    cfg->header_size = sizeof(UfoRoofPacketHeader);
     cfg->payload_size = 0;
     cfg->max_packet_size = 0;
     cfg->max_packets = 100;
@@ -86,11 +99,12 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
     cfg->drop_buffers = 0;
     cfg->path = NULL;
 
+
 	// Read configuration
     priv->parser = json_parser_new_immutable ();
     json_parser_load_from_file (priv->parser, config, &gerr);
 
-    if (gerr != NULL) {        
+    if (gerr != NULL) {
         g_propagate_prefixed_error(error, gerr, "Error parsing JSON file (%s) with ROOF configuration: ", config);
         ufo_roof_config_free(cfg);
         return NULL;
@@ -100,75 +114,124 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
 
     if (root) {
 	roof_config_node_get(hardware, root, object, "hardware");
+	roof_config_node_get(geometry, root, object, "geometry");
+	roof_config_node_get(optics, root, object, "optics");
 	roof_config_node_get(network, root, object, "network");
+	roof_config_node_get(reconstruction, root, object, "reconstruction");
 	roof_config_node_get(simulation, root, object, "simulation");
 	roof_config_node_get(performance, root, object, "performance");
     }
 
     if (hardware) {
-	// FIXME: Compute dataset size based on roof hardware
+	roof_config_node_get(cfg->n_planes, hardware, int, "planes");
+	roof_config_node_get(cfg->n_modules, hardware, int, "modules");
+	roof_config_node_get(cfg->channels_per_module, hardware, int, "channels_per_module");
+	roof_config_node_get(cfg->bit_depth, hardware, int, "bit_depth");
+
+	roof_config_node_get(cfg->samples_per_rotation, hardware, int, "samples_per_rotation");
+	roof_config_node_get(cfg->sample_rate, hardware, int, "sample_rate");
+	roof_config_node_get(cfg->imaging_rate, hardware, int, "imaging_rate");
+	
+	if ((cfg->sample_rate)||(cfg->imaging_rate)) {
+	    if ((!cfg->sample_rate)||(!cfg->imaging_rate)||(cfg->sample_rate%cfg->imaging_rate)) {
+		ufo_roof_config_free(cfg);
+		roof_new_error(error, "Invalid sample (%u) and imaging (%u) rates are specified", cfg->sample_rate, cfg->imaging_rate);
+	    }
+
+	    if ((json_object_get_member(hardware, "samples_per_rotation"))&&(cfg->samples_per_rotation != (cfg->sample_rate / cfg->imaging_rate))) {
+		ufo_roof_config_free(cfg);
+		roof_new_error(error, "The specified samples-per-rotation (%u) doesn't match   sample/imaging rates (%u / %u)", cfg->samples_per_rotation, cfg->sample_rate, cfg->imaging_rate);
+	    }
+
+	    cfg->samples_per_rotation = cfg->sample_rate / cfg->imaging_rate;
+	}
+
+	if ((cfg->bit_depth%8)||(cfg->bit_depth > 32)) {
+	    ufo_roof_config_free(cfg);
+	    roof_new_error(error, "Invalid bit-depth (%u) is configured, only 8, 16, 24, 32 is currently supported", cfg->bit_depth);
+	}
+
+	cfg->fan_projections = cfg->samples_per_rotation;
+	cfg->fan_bins = cfg->n_modules * cfg->channels_per_module;
+
+	cfg->dataset_size = cfg->fan_projections * cfg->fan_bins * (cfg->bit_depth / 8);
+	cfg->n_streams = cfg->n_modules;
+	cfg->roof_mode = TRUE;
     }
 
     if (network) {
-//	int max_packet_size = 0;
-
 	roof_config_node_get(cfg->port, network, int, "port");
 	roof_config_node_get(cfg->n_streams, network, int, "streams");
 
-	roof_config_node_get(cfg->max_packet_size, network, int, "max_packet_size");
-	    // FIXME: compute payload_size based on sample_size
 	roof_config_node_get(cfg->payload_size, network, int, "payload_size");
 	roof_config_node_get(cfg->header_size, network, int, "header_size");
-	roof_config_node_get(cfg->dataset_size, network, int, "dataset_size");
-    }
-    
-    if (performance) {
-	roof_config_node_get(cfg->max_packets, performance, int, "packets_at_once");
-	roof_config_node_get(cfg->buffer_size, performance, int, "buffer_size");
-	roof_config_node_get(cfg->drop_buffers, performance, int, "drop_buffers");
+	roof_config_node_get(cfg->max_packet_size, network, int, "max_packet_size");
+        roof_config_node_get(cfg->dataset_size, network, int, "dataset_size");
+
+	if (!cfg->payload_size) {
+	    ufo_roof_config_free(cfg);
+	    roof_new_error(error, "Packet payload and header size must be set");
+	}
+
+	if ((cfg->header_size < sizeof(UfoRoofPacketHeader))&&(!strncmp(cfg->protocol, "udp", 3))) {
+            ufo_roof_config_free(cfg);
+            roof_new_error(error, "The header with packet id (%lu bytes) is expected for un-ordered protocols", sizeof(UfoRoofPacketHeader));
+	}
+	
+	if (!cfg->dataset_size)
+	    cfg->dataset_size = cfg->payload_size;
     }
 
     if (simulation) {
 	roof_config_node_get_string(cfg->path, simulation, "path");
 	roof_config_node_get(cfg->first_file_number, simulation, int, "first_file_number");
+	roof_config_node_get(cfg->header_size, simulation, int, "header_size");
+	
+	if (!cfg->payload_size)
+	    cfg->payload_size = cfg->dataset_size;
     }
 
-	// Check configuration consistency
-    if (!cfg->payload_size) {
-        ufo_roof_config_free(cfg);
-        roof_new_error(error, "Packet size is not set");
-    }
-    
-    if ((!cfg->header_size)&&(!cfg->path)) {
-        if (!strncmp(cfg->protocol, "udp", 3)) {
-                // Error if 0 implicitely set, use default value otherwise
-            if ((network)&&(json_object_get_member(network, "header_size"))) {
-                ufo_roof_config_free(cfg);
-                roof_new_error(error, "The header with packet ids is required for un-ordered protocols");
-            } else {
-                cfg->header_size = sizeof(uint32_t);
-            }
-        }
+    if (performance) {
+	roof_config_node_get(cfg->max_packets, performance, int, "packets_at_once");
+	roof_config_node_get(cfg->buffer_size, performance, int, "buffer_size");
+	roof_config_node_get(cfg->drop_buffers, performance, int, "drop_buffers");
     }
 
+
+	// Check configuration consistency
     guint fragments_per_dataset = cfg->dataset_size / cfg->payload_size;
     guint fragments_per_stream = fragments_per_dataset / cfg->n_streams;
 
+	// Dataset should be split in an integer number of network packets (we don't expect data from different datasets in one packet at the moment)
     if ((cfg->dataset_size % cfg->payload_size)||(fragments_per_dataset%cfg->n_streams)) {
         ufo_roof_config_free(cfg);
         roof_new_error(error, "Inconsistent ROOF configuration: dataset_size=%u, packet_size=%u, data_streams=%u", cfg->dataset_size, cfg->payload_size, cfg->n_streams);
     }
 
-    if (cfg->buffer_size * fragments_per_stream < cfg->max_packets) {
-        cfg->max_packets = cfg->buffer_size * fragments_per_stream / 2;
+	// Packet should contain an integer number of complete projections (their parts provided by a single module)
+    if ((cfg->roof_mode)&&(cfg->payload_size % (cfg->channels_per_module * (cfg->bit_depth / 8)))) {
+        ufo_roof_config_free(cfg);
+        roof_new_error(error, "Inconsistent ROOF configuration: packet_size=%u, projection_size=%u (%u channels x %u bits)", cfg->payload_size, cfg->channels_per_module * (cfg->bit_depth / 8), cfg->channels_per_module, cfg->bit_depth);
     }
-    
-	// Finalize configuration
+
     if (!cfg->max_packet_size) 
-        cfg->max_packet_size = cfg->header_size + cfg->payload_size;
+	cfg->max_packet_size = cfg->header_size + cfg->payload_size;
+
+    if (hardware) {
+	if (cfg->n_modules != cfg->n_streams) {
+	    ufo_roof_config_free(cfg);
+	    roof_new_error(error, "Currently, number of ROOF modules (%u) is exepcted to be equal to number of independent data streams (%u)", cfg->n_modules, cfg->n_streams);
+	}
+
+	if (cfg->dataset_size != (cfg->fan_projections * cfg->fan_bins * cfg->bit_depth / 8)) {
+	    ufo_roof_config_free(cfg);
+	    roof_new_error(error, "Specified dataset size (%u) does not match ROOF configuration (modules: %u, channels-per-module: %u, bit-depth: %u, samples-per-rotation: %u)", cfg->dataset_size, cfg->n_modules, cfg->channels_per_module, cfg->bit_depth, cfg->samples_per_rotation);
+	}
+    }
 
-    if (!cfg->dataset_size) 
-        cfg->dataset_size = cfg->payload_size;
+    if ((cfg->buffer_size * fragments_per_stream) < cfg->max_packets) {
+        cfg->max_packets = cfg->buffer_size * fragments_per_stream / 2;
+    }
 
     if (cfg->buffer_size < 4) {
 	cfg->drop_buffers = 0;
@@ -176,5 +239,7 @@ UfoRoofConfig *ufo_roof_config_new(const char *config, GError **error) {
 	cfg->drop_buffers = cfg->buffer_size / 2;
     }
 
+    printf("dataset size: %i\n", cfg->dataset_size);
+
     return cfg;
 }
diff --git a/src/ufo-roof-config.h b/src/ufo-roof-config.h
index f90c5f3..b6ee748 100644
--- a/src/ufo-roof-config.h
+++ b/src/ufo-roof-config.h
@@ -4,30 +4,51 @@
 #include <glib.h>
 
 typedef struct {
+	// ROOF Hardware
+    gboolean			roof_mode;				// Indicates if ROOF is configured (1), otherwise only networking is implemented
+    guint			n_planes;				// Number of detector planes, ROOF module serves a ring segment from all planes in a round-robin fashion
+    guint			n_modules;				// Number of ROOF modules
+    guint			channels_per_module;			// Number of pixels in each module
+    guint			samples_per_rotation;			// Number of samples (projections) in a full fan sinogram; computed from sample_rate & image_rate if given
+    guint			sample_rate;				// Number of samples (projections) acquired per second, 0 - if unknown
+    guint			imaging_rate;				// Number of complete datasets (images) acquired per second, 0 - if unknown
+    guint			bit_depth;				// Number of bits per pixel (we currently support only multiples of 8)
+
+	// Geometry
+    guint			fan_projections;			// Number of fan projections = samples_per_rotation
+    guint			fan_bins;				// Number of fan detectors = n_modules * channels_per_module
+    guint			parallel_projections;
+    guint			parallel_bins;
+//    guint			detector_diameter;
+
+	// Optics
+
+
+	// Network Server / Reader
     gchar			*path;					// Location of data files for simmulation purposes (i.e. reading a sequence of files instead listening on the corresponding ports)
     guint                       first_file_number;                      // Indicates if the numbering of files starts at 0 or 1
     gchar			*protocol;				// Protocols: tcp, udp, tcp6, udp6, ...
     guint			port;					// First port
-    guint			n_streams;				// Number of independent data streams (expected on sequential ports)
+    guint			n_streams;				// Number of independent data streams (expected on sequential ports), by default equal to number of ROOF modules
     guint			header_size;				// Expected size of the packet header, for dgram protocols we need at least 32-bit sequence number. Defaults to uint32_t for udp* and 0 - otherwise
     guint			payload_size;				// Expected size of TCP/UDP packet (without header)
-    guint			dataset_size;				// Size of a single dataset (image, sinogram, etc.). This is real size in bytes, excluding all technical headers used in communication protocol.
+    guint			dataset_size;				// Size of a single dataset (image, sinogram, etc.). This is real size in bytes, excluding all technical headers used in communication protocol. Normally, it is computed based on ROOF hardware parameters.
+
+	// Performance parameters
+    guint                       max_packets;                            // limits maximum number of packets which are read at once
+    guint                       max_packet_size;                        // payload_size + header_size + ... (we don't care if tail is variable length provided that the complete packet does not exceed max_packet_size bytes)
+    guint			buffer_size;				// How many datasets we can buffer. There is no sense to have more than 2 for odered protocols (default), but having larger number could help for UDP if significant order disturbances are expected
+    guint			drop_buffers;				// If we are slow and lost some buffers, we may drop more than minimally necessary to catch up.
+    guint                       network_timeout;                        // Maximum time (us) to wait for data on the socket
+
 
-//?
 
 
 /*
-    guint                       pixels_per_module;
     guint                       planes_per_module;
-    guint                       samples_per_dataset;
 */    
 
-    guint                       max_packets;                            // limits maximum number of packets which are read at once
-    guint                       max_packet_size;                        // payload_size + header_size + ...?
-    guint			buffer_size;				// How many datasets we can buffer. There is no sense to have more than 2 for odered protocols (default), but having larger number could help for UDP if significant order disturbances are expected
-    guint			drop_buffers;				// If we are slow and lost some buffers, we may drop more than minimally necessary to catch up.
-    guint                       network_timeout;                        // Maximum time (us) to wait for data on the socket
-    
+
 
 
 } UfoRoofConfig;
diff --git a/src/ufo-roof.h b/src/ufo-roof.h
index d9d3a57..9303045 100644
--- a/src/ufo-roof.h
+++ b/src/ufo-roof.h
@@ -8,7 +8,7 @@
 #define UFO_ROOF_PACKET_BLOCK_HEADER(buf, cfg) ((UfoRoofPacketBlockHeader*)(((uint8_t*)buf) + cfg->max_packets * cfg->max_packet_size))
 
 typedef struct {
-    uint32_t                    packet_id;                              // Sequential Packet ID (numbered from 0)
+    uint64_t                    packet_id;                              // Sequential Packet ID (numbered from 0)
 } UfoRoofPacketHeader;
 
 typedef struct {
diff --git a/tests/config.sh b/tests/config.sh
index ecb46b6..3d5dbba 100644
--- a/tests/config.sh
+++ b/tests/config.sh
@@ -1,6 +1,9 @@
 uname -r | grep el7 &> /dev/null
 el7=$(($? == 0))
 
+arch=""
+[ $el7 -ne 0 ] && arch="64"
+
 ods_path=/mnt/ands/ods/bin-fedora/
 vma_path=/mnt/ands/
 vma_lib_path=/mnt/ands/lib64-fedora/
diff --git a/tests/roof.sh b/tests/roof-net.sh
index 12a4cfa..66faa43 100755
--- a/tests/roof.sh
+++ b/tests/roof-net.sh
@@ -5,7 +5,7 @@
 bufs=800000
 bufs=$((bufs * 4))
 
-cat roof.yaml | yq . > roof.json
+#cat roof.yaml | sed '/simulation/,$d' | yq . > roof.json
 
 ulimit -l unlimited
 echo 1000000000 > /proc/sys/kernel/shmmax	# 18446744073692774399
@@ -13,4 +13,7 @@ echo 8000 > /proc/sys/vm/nr_hugepages		# 0
 
 #VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=$bufs LD_PRELOAD=$vma_lib \
     LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib64" GI_TYPELIB_PATH="/usr/local/lib64/girepository-1.0/" \
-    python roof.py "$@"
+    python3 roof.py -c roofhw.json "$@"
+
+
+#    python3 roof.py "$@"
diff --git a/tests/roof-sim.sh b/tests/roof-sim.sh
new file mode 100755
index 0000000..4374221
--- /dev/null
+++ b/tests/roof-sim.sh
@@ -0,0 +1,14 @@
+#! /bin/bash
+
+. config.sh
+
+#cat roof.yaml | yq r - -j | jq '' | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json
+cat roof.yaml | python3 yaml2json.py | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json
+
+LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib$arch" GI_TYPELIB_PATH="/usr/local/lib$arch/girepository-1.0/" \
+    python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino-%03i.tif" -n 1 "$@"
+#    python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino-%03i.raw" -n 1 "$@"
+
+#    python3 roof.py -c roof.json -o "/home/csa/roof2_data/test_data.sino/sino%i.tif" -n 1 "$@"
+
+#    python3 roof.py -c roof.json "$@"
diff --git a/tests/roof.json b/tests/roof.json
index 784fb6d..d39a90f 100644
--- a/tests/roof.json
+++ b/tests/roof.json
@@ -1,28 +1,37 @@
 {
-  "network": {
-    "protocol": "udp",
-    "port": 4000,
-    "streams": 16,
-    "header_size": 10,
-    "payload_size": 1280,
-    "dataset_size": 1024000
-  },
-  "performance": {
-    "buffer_size": 10,
-    "packets_at_once": 100
-  },
-  "setup": {
-    "planes": 2,
-    "modules": 16,
-    "bit_depth": 16,
-    "pixels_per_module": 16,
-    "samples_per_rotation": 2000000
-  },
-  "geometry": {
-    "fan_projections": 1000,
-    "fan_detectors": 432,
-    "parallel_projections": 512,
-    "parallel_detectors": 256,
-    "detector_diameter": 216
-  }
+    "hardware": {
+        "planes": 2,
+        "modules": 16,
+        "bit_depth": 16,
+        "channels_per_module": 16,
+        "samples_per_rotation": 500
+    },
+    "geometry": {
+        "detector_diameter": [216, 216],
+        "source_diameter": [360, 365],
+        "source_angle": [270, 275],
+        "source_angle_offset": 3.2,
+        "delta_x": 500,
+        "delta_z": 1200
+    },
+    "reconstruction": {
+        "parallel_projections": 512,
+        "parallel_bins": 256
+    },
+    "network": {
+        "protocol": "udp",
+        "port": 52067,
+        "streams": 16,
+        "header_size": 8,
+        "payload_size": 800
+    },
+    "performance": {
+        "buffer_size": 10,
+        "packets_at_once": 100
+    },
+    "simulation": {
+        "path": "/home/csa/roof2_data/test_data/data_%02u.dat",
+        "first_file_number": 1,
+        "header_size": 0
+    }
 }
diff --git a/tests/roof.py b/tests/roof.py
index 831de7c..931c32f 100644
--- a/tests/roof.py
+++ b/tests/roof.py
@@ -1,35 +1,41 @@
 import gi
+import re
 import sys
 import json
 import argparse
 
+
 gi.require_version('Ufo', '0.0') 
 from gi.repository import Ufo
 from gi.repository import GObject
 
 class RoofConfig:
-    def __init__(self, config="roof.json"):
+    def __init__(self, args, config="roof.json"):
         self.streams = 1
-        
+        self.bit_depth = 8
+        self.convert = False if ((not args.output) or (re.compile('\.raw$').search(args.output))) else True
+        self.build = "raw" if args.plain else "ufo" if self.convert else "sino"
+
         with open(config) as json_file:
             cfg = json.load(json_file)
             if cfg.get("network", {}).get("streams") != None:
                 self.streams = cfg["network"]["streams"]
-            elif cfg.get("setup", {}).get("modules") != None:
+            elif cfg.get("hardware", {}).get("modules") != None:
                 self.streams = cfg["setup"]["modules"]
 
-
-config = "roof.json"
-output = None
+            if cfg.get("hardware", {}).get("bit_depth") != None:
+                self.bit_depth = cfg["hardware"]["bit_depth"]
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-c', '--config', dest="config", default="roof.json", help="ROOF configuration (JSON)")
 parser.add_argument('-o', '--output', dest="output", default=None, help="Output file")
 parser.add_argument('-n', '--number', dest="number", default=None, type=int, help="Specify number of frames to capture")
+parser.add_argument('-p', '--plain',  dest="plain",  default=False, type=bool, help="Plain network test (no ROOF structures)")
+#parser.add_argument('-r', '--raw',    dest="raw",    default=False, type=bool, help="Store raw data, ignore processed")
 args = parser.parse_args()
 
 
-cfg = RoofConfig(args.config)
+cfg = RoofConfig(args, args.config)
 
 pm  = Ufo.PluginManager()
 graph = Ufo.TaskGraph()
@@ -46,7 +52,7 @@ else:
     if args.number is None: args.number = 5
 
 build = pm.get_task('roof-build')
-build.set_properties(config=args.config, number=args.number)
+build.set_properties(config=args.config, number=args.number, build=cfg.build)
 
 for id in range(cfg.streams):
     read = pm.get_task('roof-read') 
diff --git a/tests/roof.yaml b/tests/roof.yaml
index 2a24d08..d8a1c92 100644
--- a/tests/roof.yaml
+++ b/tests/roof.yaml
@@ -1,27 +1,40 @@
+hardware:
+    planes: 2
+    modules: 16
+    bit_depth: 16
+    channels_per_module: 16
+    samples_per_rotation: 500
+#    sample_rate: 2000000
+#    imaging_rate: 2000
+geometry:
+    detector_diameter: [216, 216]
+    source_diameter: [360, 365]
+    source_angle: [270, 275]
+    source_angle_offset: 3.2
+#    source_angle_config: "path.xxx"
+    delta_x: 500
+    delta_z: 1200
+#optics:
+#   flat_field_config: "path.xxx"
+reconstruction:
+    parallel_projections: 512
+    parallel_bins: 256
 network:
     protocol: udp
-    port: 4000
+    port: 52067
     streams: 16
-    header_size: 10
-    payload_size: 1280
+    header_size: 8
+    payload_size: 800
+#    payload_size: 1280
 #    max_packet_size: 1284
-    dataset_size: 1024000
+#    dataset_size: 1024000
 performance:
     buffer_size: 10
 #    drop_buffers: 0
     packets_at_once: 100
-#simulation:
-#    first_file_number: 1
-#    path: "/mnt/fast/ROOF2/roof2-data.pumpe256/meas/data_pumpe_dyn_192.168.100_%02u.dat"
-setup:
-    planes: 2
-    modules: 16
-    bit_depth: 16
-    pixels_per_module: 16
-    samples_per_rotation: 2000000
-geometry:
-    fan_projections: 1000
-    fan_detectors: 432
-    parallel_projections: 512
-    parallel_detectors: 256
-    detector_diameter: 216
+simulation:
+    path: "/home/csa/roof2_data/test_data/data_%02u.dat"
+    first_file_number: 1
+    header_size: 0
+#    max_packet_size: 1284
+#    dataset_size: 1024000
diff --git a/tests/test_file.sh b/tests/test_file.sh
index 1eb29ca..943268f 100644
--- a/tests/test_file.sh
+++ b/tests/test_file.sh
@@ -3,13 +3,14 @@
 shopt -s extglob
 
 packet_size=1280
-packets_per_dataset=50
+packets_per_dataset=25
+packets_per_file=3150
 
 rm -f roof_test.raw
-for packet in $(seq 0 24); do
+for packet in $(seq 0 0); do
     for id in $(seq 1 16); do
-        name=$(ls -- *$id.@(fx|dat) | grep -P "_0?$id\.\w+")
-        echo "Appending packet $packet from $name "
-        dd if=$name of="roof_test.raw" bs=$packet_size count=$packets_per_dataset skip=$((packet * $packets_per_dataset))  oflag=append conv=notrunc status=none
+        name=$(ls -- *$id.@(fx|dat) | grep -P "_0?$id\.\w+" | grep -v dark | grep -v flat)
+        echo "Appending packet $packet from $name ..."
+        dd if="$name" of="roof_test.raw" bs=$packet_size count=$packets_per_dataset skip=$((packet * $packets_per_dataset))  oflag=append conv=notrunc status=none
     done
 done
diff --git a/tests/yaml2json.py b/tests/yaml2json.py
new file mode 100644
index 0000000..e97b472
--- /dev/null
+++ b/tests/yaml2json.py
@@ -0,0 +1,19 @@
+import sys
+import yaml
+import json
+from collections import OrderedDict 
+
+def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
+    class OrderedLoader(Loader):
+        pass
+    def construct_mapping(loader, node):
+        loader.flatten_mapping(node)
+        return object_pairs_hook(loader.construct_pairs(node))
+    OrderedLoader.add_constructor(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        construct_mapping)
+    return yaml.load(stream, OrderedLoader)
+
+#obj = json.load(sys.stdin, object_pairs_hook=OrderedDict)
+obj = ordered_load(sys.stdin, yaml.SafeLoader)
+print(json.dumps(obj, indent=4, sort_keys=False))