C#でファイルのMD5を高速に計算する：直列・並列・パイプライン処理の速度比較

using System.Security.Cryptography;
using System.Text;

static string CalcMd5(string path)
{
    using var md5 = MD5.Create();
    using var stream = File.OpenRead(path);
    var hash = md5.ComputeHash(stream);
    return Convert.ToHexString(hash); // 大文字16進
}

while (true)
{
    var line = Console.ReadLine();
    if (line == null) break;

    var path = line.Trim();
    if (path.Length == 0) continue;

    try
    {
        var hash = CalcMd5(path);
        Console.WriteLine($"{path},{hash}");
    }
    catch (Exception ex)
    {
        Console.Error.WriteLine($"{path},ERROR,{ex.Message}");
    }
}

// 使い方
// Measure-Command { ls -path "G:\" -Recurse -Filter "*.avi" | % { $_.FullName } | .\MD5Single.exe > output.txt }

// Days              : 0
// Hours             : 0
// Minutes           : 2
// Seconds           : 46
// Milliseconds      : 205
// Ticks             : 1662052268
// TotalDays         : 0.00192367160648148
// TotalHours        : 0.0461681185555556
// TotalMinutes      : 2.77008711333333
// TotalSeconds      : 166.2052268
// TotalMilliseconds : 166205.2268

using System.Security.Cryptography;
using System.Threading.Channels;

var pathChannel = Channel.CreateUnbounded<string>();
var dataChannel = Channel.CreateUnbounded<(string path, byte[] data)>();
var hashChannel = Channel.CreateUnbounded<(string path, string hash)>();

// --------------------
// Stage 1: stdin → path
// --------------------
var readerTask = Task.Run(async () =>
{
    while (true)
    {
        var line = Console.ReadLine();
        if (line == null) break;

        var path = line.Trim();
        if (path.Length == 0) continue;

        await pathChannel.Writer.WriteAsync(path);
    }
    pathChannel.Writer.Complete();
});

// --------------------
// Stage 2: path → file read
// --------------------
var fileReadTask = Task.Run(async () =>
{
    await foreach (var path in pathChannel.Reader.ReadAllAsync())
    {
        try
        {
            var data = await File.ReadAllBytesAsync(path);
            await dataChannel.Writer.WriteAsync((path, data));
        }
        catch (Exception ex)
        {
            await hashChannel.Writer.WriteAsync((path, $"ERROR,{ex.Message}"));
        }
    }
    dataChannel.Writer.Complete();
});

// --------------------
// Stage 3: data → MD5
// --------------------
var hashTask = Task.Run(async () =>
{
    using var md5 = MD5.Create();

    await foreach (var (path, data) in dataChannel.Reader.ReadAllAsync())
    {
        try
        {
            var hash = Convert.ToHexString(md5.ComputeHash(data));
            await hashChannel.Writer.WriteAsync((path, hash));
        }
        catch (Exception ex)
        {
            await hashChannel.Writer.WriteAsync((path, $"ERROR,{ex.Message}"));
        }
    }
    hashChannel.Writer.Complete();
});

// --------------------
// Stage 4: stdout
// --------------------
var writerTask = Task.Run(async () =>
{
    await foreach (var (path, hash) in hashChannel.Reader.ReadAllAsync())
    {
        Console.WriteLine($"{path},{hash}");
    }
});

await Task.WhenAll(readerTask, fileReadTask, hashTask, writerTask);

/*
使い方
Measure-Command { ls -path "G:\" -Recurse -Filter "*.avi" | % { $_.FullName } | .\MD5Pipeline.exe > output.txt }

Days              : 0
Hours             : 0
Minutes           : 1
Seconds           : 53
Milliseconds      : 914
Ticks             : 1139147432
TotalDays         : 0.00131845767592593
TotalHours        : 0.0316429842222222
TotalMinutes      : 1.89857905333333
TotalSeconds      : 113.9147432
TotalMilliseconds : 113914.7432
*/

using System.Collections.Concurrent;
using System.Security.Cryptography;
using System.Text;

static string CalcMd5(string path)
{
    using var md5 = MD5.Create();
    using var stream = File.OpenRead(path);
    var hash = md5.ComputeHash(stream);
    return Convert.ToHexString(hash); // 大文字16進
}

var paths = new List<string>();

while (true)
{
    var line = Console.ReadLine();
    if (line == null) break;

    var path = line.Trim();
    if (path.Length > 0)
        paths.Add(path);
}

var results = new ConcurrentQueue<string>();

Parallel.ForEach(paths, path =>
{
    try
    {
        var hash = CalcMd5(path);
        results.Enqueue($"{path},{hash}");
    }
    catch (Exception ex)
    {
        results.Enqueue($"{path},ERROR,{ex.Message}");
    }
});

foreach (var line in results)
{
    Console.WriteLine(line);
}

/*
実行例
Measure-Command { ls -path "G:\" -Recurse -Filter "*.avi" | % { $_.FullName } | .\MD5Parallel.exe > output.txt }

Days              : 0
Hours             : 0
Minutes           : 0
Seconds           : 51
Milliseconds      : 863
Ticks             : 518630308
TotalDays         : 0.000600266560185185
TotalHours        : 0.0144063974444444
TotalMinutes      : 0.864383846666667
TotalSeconds      : 51.8630308
TotalMilliseconds : 51863.0308
*/