Files
GSS2Rework/GSS2.Core/Analysis/AmineContent/AnalyzerService.cs
T
2026-05-13 15:47:31 +03:00

533 lines
21 KiB
C#

using Microsoft.Extensions.Logging;
using Microsoft.ML;
using Microsoft.ML.Data;
using OpenCvSharp;
using GSS2.Core.Extensions;
using GSS2.Core.Analysis.AmineContent.Database.Calibration;
namespace GSS2.Core.Analysis.AmineContent;
public partial class AnalyzerService
{
//TODO: Перенести все гиперпараметры в appsettings.json
public const int CHANNELS_COUNT = 3;
public const int IMAGES_COUNT = 12;
public const int FEATURES_LENGTH = (CHANNELS_COUNT + 3) * IMAGES_COUNT;
public const int EFFECTIVE_IMAGE_SIZE = 200;
public const int ANALYSIS_IMAGE_SIZE = 1000;
public const int SEPARATOR_ROI_THRESHOLD = 5;
public const float CONTOUR_MAX_SIZE = 100;
public const float CONTOUR_MAX_AREA = (float)Math.PI * (CONTOUR_MAX_SIZE * CONTOUR_MAX_SIZE) / 4.0f;
public const float CONTOUR_MIN_SIZE = 12;
public const float CONTOUR_MIN_AREA = (float)Math.PI * (CONTOUR_MIN_SIZE * CONTOUR_MIN_SIZE) / 4.0f / 4.0f;
public const int SEPARATOR_X = 519;
public const int SEPARATOR_Y = 17;
public const int SEPARATOR_WIDTH = 2989;
public const int SEPARATOR_HEIGHT = 2985;
private class ClusterizationData
{
[ColumnName("Label")]
public required string Label { get; set; }
[ColumnName("Features")]
[VectorType(FEATURES_LENGTH)]
public required float[] Features { get; set; }
}
private class ClusterizationPrediction
{
[ColumnName("PredictedLabel")]
public string Label { get; set; }
public float Probability { get; set; }
public float[] Score { get; set; }
public ClusterizationPrediction()
{
Label = "";
Probability = 0;
Score = new float[2];
}
}
private class RegressionData
{
[ColumnName("Label")]
public required float Value { get; set; }
[ColumnName("Features")]
[VectorType(FEATURES_LENGTH)]
public required float[] Features { get; set; }
}
private class RegressionPrediction
{
[ColumnName("Score")]
public float Value { get; set; }
public RegressionPrediction()
{
Value = -1;
}
}
private readonly ILogger<AnalyzerService> _logger;
private readonly CalibrationContext _calibrationContext;
private MLContext? _ml = null;
private PredictionEngine<ClusterizationData, ClusterizationPrediction>? _clusterizationEngine = null;
private PredictionEngine<RegressionData, RegressionPrediction>? _regressionEngine = null;
public bool Initialized { get; private set; } = false;
public AnalyzerService(ILogger<AnalyzerService> logger, CalibrationContext calibrationContext)
{
_logger = logger;
_calibrationContext = calibrationContext;
}
public async Task Initialize(string? brand = null, CancellationToken cancellationToken = default)
{
Initialized = false;
_logger.LogInformation("Инициализация");
_ml = new MLContext();
_ml.Log += (_, ea) =>
{
if (ea.Kind >= Microsoft.ML.Runtime.ChannelMessageKind.Info)
_logger.LogInformation("ML: [{}] {}", ea.Kind, ea.Message);
};
try
{
await InitializeMlClusterizationEngine(cancellationToken);
await InitializeMlRegressionEngine(brand, cancellationToken);
}
catch (Exception ex)
{
_logger.LogError(ex, "Исключение во время инициализации моделей");
return;
}
Initialized = true;
}
private async Task InitializeMlClusterizationEngine(CancellationToken cancellationToken)
{
_logger.LogInformation("Инициализация модели кластеризации");
var separatorPrefix = "SEPARATOR";
var samplePrefix = "SAMPLE";
if (_ml is null)
throw new InvalidOperationException();
cancellationToken.ThrowIfCancellationRequested();
_logger.LogInformation("Создание конвейера");
var pipeline = _ml.Transforms.Conversion.MapValueToKey("Label")
.Append(_ml.MulticlassClassification.Trainers.LbfgsMaximumEntropy())
.Append(_ml.Transforms.Conversion.MapKeyToValue("PredictedLabel"));
cancellationToken.ThrowIfCancellationRequested();
_logger.LogInformation("Подготовка данных");
var trainDataSet = _calibrationContext.VectorRecords.Select(r =>
new ClusterizationData
{
Label = r.SeparatorRecord != null ?
$"{separatorPrefix}|{r.SeparatorRecord.Type}" :
r.SampleRecord != null ?
$"{samplePrefix}|{r.SampleRecord.SampleBrand}" :
"",
Features = r.Values
})
.ToList();
var trainData = _ml.Data.LoadFromEnumerable(trainDataSet);
cancellationToken.ThrowIfCancellationRequested();
_logger.LogInformation("Обучение модели");
var model = pipeline.Fit(trainData);
_clusterizationEngine = _ml.Model.CreatePredictionEngine<ClusterizationData, ClusterizationPrediction>(model);
model.Dispose();
}
private async Task InitializeMlRegressionEngine(string? brand, CancellationToken cancellationToken)
{
_logger.LogInformation("Инициализация модели регрессии для марки \"{}\"", brand);
if (_ml is null)
throw new InvalidOperationException();
cancellationToken.ThrowIfCancellationRequested();
_logger.LogInformation("Создание конвейера");
var pipeline = _ml.Regression.Trainers.Sdca();
cancellationToken.ThrowIfCancellationRequested();
_logger.LogInformation("Подготовка данных");
var trainDataSet = _calibrationContext.VectorRecords
.Where(r => r.SampleRecord != null)
.Where(r => brand == null || r.SampleRecord!.SampleBrand == brand)
.Select(r =>
new RegressionData
{
Value = (float)(
r.SampleRecord!.MeasuredContent >= 0 ?
r.SampleRecord!.MeasuredContent :
r.SampleRecord!.MixtureActualRate >= 0 ?
r.SampleRecord!.MixtureActualRate :
r.SampleRecord!.MixtureNormalRate
),
Features = r.Values
})
.ToList();
var trainData = _ml.Data.LoadFromEnumerable(trainDataSet);
cancellationToken.ThrowIfCancellationRequested();
_logger.LogInformation("Обучение модели");
var model = pipeline.Fit(trainData);
_regressionEngine = _ml.Model.CreatePredictionEngine<RegressionData, RegressionPrediction>(model);
model.Dispose();
}
public async Task<(Mat avgImage, Rect roi, Mat mask, IEnumerable<Point[]> contours, Mat result, string separatorType, string sampleBrand)?> Analyze(IEnumerable<ImageData> imagesData)
{
if (!Initialized)
return null;
if (imagesData.Count() != IMAGES_COUNT)
return null;
try
{
var segmentationResult = await Segment(imagesData);
if (segmentationResult is null)
return null;
var (avgImage, roi, mask, contours, separatorType, sampleBrand) = segmentationResult.Value;
var images = new List<(Mat mat, double visible, double uv365, double uv254)>();
foreach (var imageData in imagesData
.OrderBy(i => i.VisibleIntensity)
.ThenBy(i => i.Uv365Intensity)
.ThenBy(i => i.Uv254Intensity))
images.Add((
LoadAndPrepare(imageData.Path, roi, ANALYSIS_IMAGE_SIZE),
imageData.VisibleIntensity,
imageData.Uv365Intensity,
imageData.Uv254Intensity
));
var result = new Mat(new Size(ANALYSIS_IMAGE_SIZE, ANALYSIS_IMAGE_SIZE), MatType.CV_32FC1, new Scalar(0));
var values = new float[ANALYSIS_IMAGE_SIZE * ANALYSIS_IMAGE_SIZE];
for (var row = 0; row < ANALYSIS_IMAGE_SIZE; row++)
for (var col = 0; col < ANALYSIS_IMAGE_SIZE; col++)
{
if (mask.At<byte>(row, col) != 255)
continue;
var vector = new float[FEATURES_LENGTH];
foreach (var (i, image) in images.Enumerate())
{
vector[i * (CHANNELS_COUNT + 3) + 0] = (float)image.visible;
vector[i * (CHANNELS_COUNT + 3) + 1] = (float)image.uv365;
vector[i * (CHANNELS_COUNT + 3) + 2] = (float)image.uv254;
var color = image.mat.At<Vec3b>(row, col);
vector[i * (CHANNELS_COUNT + 3) + 3] = color.Item0 / 255.0f;
vector[i * (CHANNELS_COUNT + 3) + 4] = color.Item1 / 255.0f;
vector[i * (CHANNELS_COUNT + 3) + 5] = color.Item2 / 255.0f;
}
var prediction = _regressionEngine?.Predict(new RegressionData
{
Value = -1,
Features = vector
});
if (prediction is null)
continue;
result.Set<float>(row, col, prediction.Value);
values[row * ANALYSIS_IMAGE_SIZE + col] = prediction.Value;
}
var oldMean = Cv2.Mean(result, mask).Val0;
values.Sort();
float p999 = values[(int)(values.Length * 0.999)];
Cv2.Threshold(result, result, p999, p999, ThresholdTypes.Trunc);
var newMean = Cv2.Mean(result, mask).Val0;
var scale = oldMean / newMean;
Cv2.ConvertScaleAbs(result, result, scale);
return (avgImage, roi, mask, contours, result, separatorType, sampleBrand);
}
catch (Exception ex)
{
_logger.LogError(ex, "Исключение при проведении анализа");
return null;
}
}
private async Task<(Mat avgImage, Rect roi, Mat mask, IEnumerable<Point[]> contours, string separatorType, string sampleBrand)?> Segment(IEnumerable<ImageData> imagesData)
{
var separatorPrefix = "SEPARATOR";
var samplePrefix = "SAMPLE";
if (imagesData.Count() != IMAGES_COUNT)
return null;
var avgImage = CalculateAverageImage(imagesData.Select(i => i.Path));
if (avgImage is null)
return null;
var roi = CalculateSeparatorRoi(avgImage);
if (roi is null)
{
avgImage.Release();
return null;
}
avgImage = new Mat(avgImage, roi.Value);
Cv2.Resize(avgImage, avgImage, new Size(ANALYSIS_IMAGE_SIZE, ANALYSIS_IMAGE_SIZE));
var images = new List<(Mat mat, double visible, double uv365, double uv254)>();
foreach (var imageData in imagesData
.OrderBy(i => i.VisibleIntensity)
.ThenBy(i => i.Uv365Intensity)
.ThenBy(i => i.Uv254Intensity))
images.Add((
LoadAndPrepare(imageData.Path, roi.Value, ANALYSIS_IMAGE_SIZE),
imageData.VisibleIntensity,
imageData.Uv365Intensity,
imageData.Uv254Intensity
));
var separatorMask = new Mat(new Size(ANALYSIS_IMAGE_SIZE, ANALYSIS_IMAGE_SIZE), MatType.CV_8UC1, new Scalar(0));
Cv2.Circle(separatorMask, ANALYSIS_IMAGE_SIZE / 2, ANALYSIS_IMAGE_SIZE / 2, ANALYSIS_IMAGE_SIZE / 2, new Scalar(255), -1);
var result = new Mat(new Size(ANALYSIS_IMAGE_SIZE, ANALYSIS_IMAGE_SIZE), MatType.CV_16UC1, new Scalar(0));
var keyMap = new Dictionary<string, ushort>();
ushort lastKey = 0;
for (var row = 0; row < ANALYSIS_IMAGE_SIZE; row++)
for (var col = 0; col < ANALYSIS_IMAGE_SIZE; col++)
{
if (separatorMask.At<byte>(row, col) != 255)
continue;
var vector = new float[FEATURES_LENGTH];
foreach (var (i, image) in images.Enumerate())
{
vector[i * (CHANNELS_COUNT + 3) + 0] = (float)image.visible;
vector[i * (CHANNELS_COUNT + 3) + 1] = (float)image.uv365;
vector[i * (CHANNELS_COUNT + 3) + 2] = (float)image.uv254;
var color = image.mat.At<Vec3b>(row, col);
vector[i * (CHANNELS_COUNT + 3) + 3] = color.Item0 / 255.0f;
vector[i * (CHANNELS_COUNT + 3) + 4] = color.Item1 / 255.0f;
vector[i * (CHANNELS_COUNT + 3) + 5] = color.Item2 / 255.0f;
}
var prediction = _clusterizationEngine?.Predict(new ClusterizationData
{
Label = "",
Features = vector
});
if (prediction is null)
continue;
if (!keyMap.ContainsKey(prediction.Label))
keyMap.Add(prediction.Label, ++lastKey);
result.Set<ushort>(row, col, keyMap[prediction.Label]);
}
foreach (var image in images)
image.mat.Release();
var valueMap = keyMap.ToDictionary(kv => kv.Value, kv => kv.Key);
var frequency = new Dictionary<string, int>();
for (var row = 0; row < ANALYSIS_IMAGE_SIZE; row++)
for (var col = 0; col < ANALYSIS_IMAGE_SIZE; col++)
{
var value = result.At<ushort>(row, col);
if (value == 0)
continue;
if (!frequency.ContainsKey(valueMap[value]))
frequency.Add(valueMap[value], 0);
frequency[valueMap[value]] += 1;
if (valueMap[value].StartsWith(separatorPrefix))
result.Set<ushort>(row, col, 0);
if (valueMap[value].StartsWith(samplePrefix))
result.Set<ushort>(row, col, ushort.MaxValue);
}
result.ConvertTo(result, MatType.CV_8UC1, 1 / 255.0);
Cv2.MorphologyEx(result, result, MorphTypes.Close, Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3)), iterations: 2);
Cv2.MorphologyEx(result, result, MorphTypes.Open, Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3)), iterations: 2);
Cv2.Dilate(result, result, Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3)));
Cv2.GaussianBlur(result, result, new Size(5, 5), 1);
Cv2.Threshold(result, result, 127, 255, ThresholdTypes.Binary);
Cv2.FindContours(result, out var contours, out _, RetrievalModes.List, ContourApproximationModes.ApproxSimple);
result = new Mat(new Size(ANALYSIS_IMAGE_SIZE, ANALYSIS_IMAGE_SIZE), MatType.CV_8UC1, new Scalar(0));
Cv2.DrawContours(result, contours.Where(ContourIsValid), -1, new Scalar(255), -1);
return (
avgImage,
roi.Value,
result,
contours.Where(ContourIsValid),
frequency
.Where(kv => kv.Key.StartsWith(separatorPrefix))
.MaxBy(kv => kv.Value)
.Key
.Substring(separatorPrefix.Count() + 1),
frequency
.Where(kv => kv.Key.StartsWith(samplePrefix))
.MaxBy(kv => kv.Value)
.Key
.Substring(samplePrefix.Count() + 1)
);
}
public static (Mat avgImage, Rect roi, List<float[]> vectors)? GetFeatureVectors(IEnumerable<ImageData> imagesData)
{
if (imagesData.Count() != IMAGES_COUNT)
return null;
var avgImage = CalculateAverageImage(imagesData.Select(i => i.Path));
if (avgImage is null)
return null;
var roi = CalculateSeparatorRoi(avgImage);
if (roi is null)
return null;
var images = new List<(Mat mat, double visible, double uv365, double uv254)>();
foreach (var imageData in imagesData
.OrderBy(i => i.VisibleIntensity)
.ThenBy(i => i.Uv365Intensity)
.ThenBy(i => i.Uv254Intensity))
images.Add((
LoadAndPrepare(imageData.Path, roi.Value),
imageData.VisibleIntensity,
imageData.Uv365Intensity,
imageData.Uv254Intensity
));
var separatorMask = new Mat(new Size(EFFECTIVE_IMAGE_SIZE, EFFECTIVE_IMAGE_SIZE), MatType.CV_8UC1, new Scalar(0));
Cv2.Circle(separatorMask, EFFECTIVE_IMAGE_SIZE / 2, EFFECTIVE_IMAGE_SIZE / 2, EFFECTIVE_IMAGE_SIZE / 2, new Scalar(255), -1);
var vectors = new List<float[]>();
for (var row = 0; row < EFFECTIVE_IMAGE_SIZE; row++)
for (var col = 0; col < EFFECTIVE_IMAGE_SIZE; col++)
{
if (separatorMask.At<byte>(row, col) != 255)
continue;
var vector = new float[FEATURES_LENGTH];
foreach (var (i, image) in images.Enumerate())
{
vector[i * (CHANNELS_COUNT + 3) + 0] = (float)image.visible;
vector[i * (CHANNELS_COUNT + 3) + 1] = (float)image.uv365;
vector[i * (CHANNELS_COUNT + 3) + 2] = (float)image.uv254;
var color = image.mat.At<Vec3b>(row, col).ToVec3f() / 255.0f;
vector[i * (CHANNELS_COUNT + 3) + 3] = color.Item0;
vector[i * (CHANNELS_COUNT + 3) + 4] = color.Item1;
vector[i * (CHANNELS_COUNT + 3) + 5] = color.Item2;
}
vectors.Add(vector);
}
foreach (var (mat, _, _, _) in images)
mat.Release();
return (avgImage, roi.Value, vectors);
}
private static Mat LoadAndPrepare(string imagePath, Rect roi, int size = EFFECTIVE_IMAGE_SIZE)
{
var image = Cv2.ImRead(imagePath);
image = new Mat(image, roi);
Cv2.GaussianBlur(image, image, new Size(5, 5), 1);
Cv2.Resize(image, image, new Size(size, size));
return image;
}
public static Mat? CalculateAverageImage(IEnumerable<string> imagePaths)
{
if (imagePaths.Count() == 0)
return null;
Mat avgImage = new Mat();
foreach (var imagePath in imagePaths)
{
var image = Cv2.ImRead(imagePath);
if (image.Channels() != 3)
return null;
if (avgImage.Empty())
avgImage = new Mat(image.Size(), MatType.CV_16UC3, new Scalar(0, 0, 0));
if (image.Size().Width != avgImage.Size().Width ||
image.Size().Height != avgImage.Size().Height)
return null;
Cv2.Add(image, avgImage, avgImage, dtype: MatType.CV_16U);
image.Release();
}
avgImage.ConvertTo(avgImage, MatType.CV_8UC(avgImage.Channels()), 1f / imagePaths.Count());
return avgImage;
}
public static Rect? CalculateSeparatorRoi(Mat avgImage)
{
if (avgImage.Empty())
return null;
return new Rect(SEPARATOR_X, SEPARATOR_Y, SEPARATOR_WIDTH, SEPARATOR_HEIGHT);
}
// Этот метод конечно хорош, но проблема с тем, что при калибровке на чёрном продукте
// он выдаёт очень обрезает часть полезного изображения из-за неравномерности освещения
// public static Rect? CalculateSeparatorRoi(Mat avgImage)
// {
// if (avgImage.Empty())
// return null;
// Mat avgImageClone = new Mat();
// Mat mask = new Mat();
// Rect? roi = null;
// avgImage.CopyTo(avgImageClone);
// avgImageClone = avgImageClone.CvtColor(ColorConversionCodes.BGR2GRAY);
// Cv2.Threshold(avgImageClone, mask, SEPARATOR_ROI_THRESHOLD, 255, ThresholdTypes.Binary);
// Cv2.FindContours(mask, out var contours, out _, RetrievalModes.List, ContourApproximationModes.ApproxSimple);
// var maxContour = contours.MaxBy(c => Cv2.ContourArea(c));
// if (maxContour is null)
// return null;
// roi = Cv2.BoundingRect(maxContour);
// avgImageClone.Release();
// mask.Release();
// return roi;
// }
private static bool ContourIsValid(Point[] contour)
{
var bbox = Cv2.BoundingRect(contour);
var area = Cv2.ContourArea(contour);
return CONTOUR_MIN_SIZE < bbox.Width && CONTOUR_MIN_SIZE < bbox.Height && CONTOUR_MIN_AREA < area &&
CONTOUR_MAX_SIZE > bbox.Width && CONTOUR_MAX_SIZE > bbox.Height && CONTOUR_MAX_AREA > area;
}
}