comprehensive update

1) remove unused components: camera view (due it causes segmentation faults), compute resources and all related, illuminator controller (due in useless without camera view), image storage service, temperature and humidity service
2) database schema - reduce tables references, features storing in records themselves in compressed form, add records creation and editing date and time, add separator comment column
3) analysis - rework of pipeline and ui, now database storing only raw data and all display values calculated from it
4) lights service - add reconnection if disconnected
5) add width and height command line arguments
6) fix some typos and other issues

GSS2.Core/Analysis/AmineContent/AnalyzerService.cs

@@ -84,6 +84,8 @@ public partial class AnalyzerService
     private PredictionEngine<RegressionData, RegressionPrediction>? _regressionEngine = null;
 
     public bool Initialized { get; private set; } = false;
+    public int InitializedBrandId { get; private set; } = 0;
+    public int InitializedSeparatorId { get; private set; } = 0;
 
     public AnalyzerService(ILogger<AnalyzerService> logger, Context Context)
     {
@@ -93,6 +95,14 @@ public partial class AnalyzerService
 
     public async Task Initialize(BrandRecord brand, SeparatorRecord separator, CancellationToken cancellationToken = default)
     {
+        // TODO: Повторная инициализация если изменились калибровочные данные
+
+        if (Initialized &&
+            InitializedBrandId == brand.Id &&
+            InitializedSeparatorId == separator.Id
+        )
+            return;
+
         Initialized = false;
 
         _logger.LogInformation("Инициализация");
@@ -116,6 +126,8 @@ public partial class AnalyzerService
         }
 
         Initialized = true;
+        InitializedBrandId = brand.Id;
+        InitializedSeparatorId = separator.Id;
     }
     private async Task InitializeMlClusterizationEngine(BrandRecord brand, SeparatorRecord separator, CancellationToken cancellationToken)
     {
@@ -126,35 +138,49 @@ public partial class AnalyzerService
         cancellationToken.ThrowIfCancellationRequested();
 
         _logger.LogInformation("Подготовка данных");
-        var sampleRecordIds = _context.SampleRecords
-            .Where(r => r.BrandId == brand.Id)
-            .Select(r => r.Id);
-        _logger.LogInformation("Найдено калибровочных записей проб для марки: {}", sampleRecordIds.Count());
-        if (sampleRecordIds.Count() == 0)
+        var sampleRecords = _context.SampleRecords
+            .Where(r => r.BrandId == brand.Id);
+
+        _logger.LogInformation("Найдено калибровочных записей проб для марки: {}", sampleRecords.Count());
+        if (sampleRecords.Count() == 0)
             throw new InvalidDataException($"Не найдено калибровочных записей для марки: \"{brand.BrandName} - {brand.MixtureName}\"");
         cancellationToken.ThrowIfCancellationRequested();
 
-        var separatorDataSet = _context.FeatureRecords
-            .Where(r => r.SeparatorRecordId == separator.Id)
-            .Select(r => new ClusterizationData
+        if (separator.Features.Count() == 0)
+            throw new InvalidDataException($"Не найдено калибровочных признаков для сепаратора: \"{separator.Type}\"");
+        if (separator.Features.Count() % FEATURES_LENGTH != 0)
+            throw new InvalidDataException($"Неожидаемое количество калибровочных признаков для сепаратора: \"{separator.Type}\"");
+        var separatorDataSet = separator.Features.Chunk(FEATURES_LENGTH)
+            .Select(v => new ClusterizationData
             {
                 Label = "separator",
-                Features = r.Values.ToArray()
-            });
+                Features = v.ToArray()
+            })
+            .Shuffle();
+
         _logger.LogInformation("Загружено векторов для сепаратора: {}", separatorDataSet.Count());
         if (separatorDataSet.Count() == 0)
             throw new InvalidDataException($"Не найдено калибровочных данных для сепаратора: \"{separator.Type}\"");
         cancellationToken.ThrowIfCancellationRequested();
 
-        var sampleDataSet = _context.FeatureRecords
-            .Where(r => r.SeparatorRecordId != 0)
-            .Where(r => sampleRecordIds.Contains(r.SampleRecordId))
-            .Select(r => new ClusterizationData
+        foreach (var sampleRecord in sampleRecords)
+        {
+            if (sampleRecord.Features.Count() == 0)
+                _logger.LogError("Не найдено калибровочных признаков для пробы: \"{}\"", sampleRecord.SampleName);
+            if (sampleRecord.Features.Count() % FEATURES_LENGTH != 0)
+                _logger.LogError("Неожидаемое количество калибровочных признаков для пробы: \"{}\"", sampleRecord.SampleName);
+        }
+
+        var sampleDataSet = sampleRecords
+            .Where(r => r.Features.Count() > 0 && r.Features.Count() % FEATURES_LENGTH == 0)
+            .SelectMany(r => r.Features.Chunk(FEATURES_LENGTH))
+            .Select(f => new ClusterizationData
             {
                 Label = "sample",
-                Features = r.Values.ToArray()
+                Features = f.ToArray()
             })
             .Shuffle();
+
         _logger.LogInformation("Загружено векторов для пробы: {}", sampleDataSet.Count());
         if (separatorDataSet.Count() == 0)
             throw new InvalidDataException($"Не найдено калибровочных данных для марки: \"{brand.BrandName} - {brand.MixtureName}\"");
@@ -187,32 +213,40 @@ public partial class AnalyzerService
         cancellationToken.ThrowIfCancellationRequested();
 
         _logger.LogInformation("Подготовка данных");
-        var sampleRecordIds = _context.SampleRecords
-            .Where(r => r.BrandId == brand.Id)
-            .Select(r => r.Id);
-        _logger.LogInformation("Найдено калибровочных записей проб для марки: {}", sampleRecordIds.Count());
-        if (sampleRecordIds.Count() == 0)
+        var sampleRecords = _context.SampleRecords
+            .Where(r => r.BrandId == brand.Id);
+
+        _logger.LogInformation("Найдено калибровочных записей проб для марки: {}", sampleRecords.Count());
+        if (sampleRecords.Count() == 0)
             throw new InvalidDataException($"Не найдено калибровочных записей для марки: \"{brand.BrandName} - {brand.MixtureName}\"");
         cancellationToken.ThrowIfCancellationRequested();
 
+        foreach (var sampleRecord in sampleRecords)
+        {
+            if (sampleRecord.Features.Count() == 0)
+                _logger.LogError("Не найдено калибровочных признаков для пробы: \"{}\"", sampleRecord.SampleName);
+            if (sampleRecord.Features.Count() % FEATURES_LENGTH != 0)
+                _logger.LogError("Неожидаемое количество калибровочных признаков для пробы: \"{}\"", sampleRecord.SampleName);
+        }
+
         _logger.LogInformation("Подготовка данных");
-        var trainDataSet = _context.FeatureRecords
-            .Where(r => r.SeparatorRecordId != 0)
-            .Where(r => sampleRecordIds.Contains(r.SampleRecordId))
-            .ToList()
-            .Select(r =>
-            {
-                var sample = _context.SampleRecords.First(s => s.Id == r.SampleRecordId);
-                return new RegressionData
-                {
-                    Value = (float)(
-                        sample.MeasuredContent >= 0 ? sample.MeasuredContent :
-                        sample.MixtureActualRate
-                    ),
-                    Features = r.Values.ToArray()
-                };
-            })
-            .Shuffle();
+        var trainDataSet = sampleRecords
+            .Where(r => r.Features.Count() > 0 && r.Features.Count() % FEATURES_LENGTH == 0)
+            .Where(r => r.MeasuredContent >= 0 || r.MixtureActualRate >= 0)
+            .SelectMany(r =>
+                r.Features
+                    .Chunk(FEATURES_LENGTH)
+                    .Select(f =>
+                        new RegressionData
+                        {
+                            Value = (float)(r.MeasuredContent >= 0 ? r.MeasuredContent : r.MixtureActualRate),
+                            Features = f.ToArray()
+                        }
+                )
+            )
+            .Shuffle()
+            .ToList();
+
         _logger.LogInformation("Загружено векторов для обучения: {}", trainDataSet.Count());
         if (trainDataSet.Count() == 0)
             throw new InvalidDataException($"Не найдено калибровочных данных для марки: \"{brand.BrandName} - {brand.MixtureName}\"");
@@ -232,13 +266,15 @@ public partial class AnalyzerService
         Vector<double> meanVector = matrix.ColumnSums() / matrix.RowCount;
         Matrix<double> covarianceMatrix = MatrixCovariance(matrix, meanVector);
         Matrix<double> invCovarianceMatrix = covarianceMatrix.Inverse();
-        trainDataSet = trainDataSet.Where((data, index) =>
-        {
-            var row = matrix.Row(index);
-            double distance = CalculateMahalanobis(row, meanVector, invCovarianceMatrix);
+        trainDataSet = trainDataSet
+            .Where((data, index) =>
+            {
+                var row = matrix.Row(index);
+                double distance = CalculateMahalanobis(row, meanVector, invCovarianceMatrix);
 
-            return distance <= FILTERING_THRESHOLD;
-        }).ToList();
+                return distance <= FILTERING_THRESHOLD;
+            })
+            .ToList();
         _logger.LogInformation("Векторов для обучения после фильтрации: {}", trainDataSet.Count());
         var trainData = _ml.Data.LoadFromEnumerable(trainDataSet);
         cancellationToken.ThrowIfCancellationRequested();