Merge remote-tracking branch 'origin/main' into #207-DE-Fiskaly-Split…

…-TAR-export
fiskaltrust · May 16, 2024 · 85974ae · 85974ae
2 parents a62281f + 05dfad4
commit 85974ae
Show file tree

Hide file tree

Showing 12 changed files with 105 additions and 128 deletions.
diff --git a/queue/src/fiskaltrust.Middleware.Localization.QueueDE/JournalProcessorDE.cs b/queue/src/fiskaltrust.Middleware.Localization.QueueDE/JournalProcessorDE.cs
@@ -80,6 +80,10 @@ public async IAsyncEnumerable<JournalResponse> ProcessAsync(JournalRequest reque
             _logger.LogDebug($"Processing JournalRequest for DE (Type: {request.ftJournalType:X}");
             if (request.ftJournalType == (long) JournalTypes.TarExportFromTSE)
             {
+                if (request.MaxChunkSize == 0)
+                {
+                    request.MaxChunkSize = _middlewareConfiguration.TarFileChunkSize;
+                }
                 await foreach (var value in ProcessTarExportFromTSEAsync(request).ConfigureAwait(false))
                 {
                     yield return value;
@@ -151,42 +155,55 @@ private async IAsyncEnumerable<JournalResponse> ProcessTarExportFromTSEAsync(Jou
         {
             var exportSession = await _deSSCDProvider.Instance.StartExportSessionAsync(new StartExportSessionRequest()).ConfigureAwait(false);
             var sha256CheckSum = "";
-            using (var memoryStream = new MemoryStream())
+
+            byte[] chunk;
+            var response = new JournalResponse();
+            try
             {
-                ExportDataResponse export;
-                do
+                using (var stream = new FileStream(exportSession.TokenId, FileMode.Create, FileAccess.ReadWrite))
                 {
-                    export = await _deSSCDProvider.Instance.ExportDataAsync(new ExportDataRequest
-                    {
-                        TokenId = exportSession.TokenId,
-                        MaxChunkSize = request.MaxChunkSize
-                    }).ConfigureAwait(false);
-                    if (!export.TotalTarFileSizeAvailable)
+                    ExportDataResponse export;
+                    do
                     {
-                        await Task.Delay(TimeSpan.FromMilliseconds(100)).ConfigureAwait(false);
-                    }
-                    else
-                    {
-                        var chunk = Convert.FromBase64String(export.TarFileByteChunkBase64);
-                        memoryStream.Write(chunk, 0, chunk.Length);
-                        yield return new JournalResponse
+                        export = await _deSSCDProvider.Instance.ExportDataAsync(new ExportDataRequest
                         {
-                            Chunk = chunk.ToList()
-                        };
-                    }
-                } while (!export.TarFileEndOfFile);
-                sha256CheckSum = Convert.ToBase64String(SHA256.Create().ComputeHash(memoryStream.ToArray()));
-            }
+                            TokenId = exportSession.TokenId,
+                            MaxChunkSize = request.MaxChunkSize
+                        }).ConfigureAwait(false);
+                        if (!export.TotalTarFileSizeAvailable)
+                        {
+                            await Task.Delay(TimeSpan.FromMilliseconds(100)).ConfigureAwait(false);
+                        }
+                        else
+                        {
+                            chunk = Convert.FromBase64String(export.TarFileByteChunkBase64);
+                            stream.Write(chunk, 0, chunk.Length);
+                            response.Chunk = chunk.ToList();
+                            yield return response;
+                        }
+                    } while (!export.TarFileEndOfFile);
+                    using var sha256 = SHA256.Create();
+                    stream.Position = 0;
+                    sha256CheckSum = Convert.ToBase64String(sha256.ComputeHash(stream));
+                }
 
-            var endSessionRequest = new EndExportSessionRequest
-            {
-                TokenId = exportSession.TokenId,
-                Sha256ChecksumBase64 = sha256CheckSum
-            };
-            var endExportSessionResult = await _deSSCDProvider.Instance.EndExportSessionAsync(endSessionRequest).ConfigureAwait(false);
-            if (!endExportSessionResult.IsValid)
+                var endSessionRequest = new EndExportSessionRequest
+                {
+                    TokenId = exportSession.TokenId,
+                    Sha256ChecksumBase64 = sha256CheckSum
+                };
+                var endExportSessionResult = await _deSSCDProvider.Instance.EndExportSessionAsync(endSessionRequest).ConfigureAwait(false);
+                if (!endExportSessionResult.IsValid)
+                {
+                    throw new Exception("The TAR file export was not successful.");
+                }
+            }
+            finally
             {
-                throw new Exception("The TAR file export was not successful.");
+                if (File.Exists(exportSession.TokenId))
+                {
+                    File.Delete(exportSession.TokenId);
+                }
             }
             yield break;
         }

diff --git a/queue/src/fiskaltrust.Middleware.Localization.QueueDE/Services/TarFileExportService.cs b/queue/src/fiskaltrust.Middleware.Localization.QueueDE/Services/TarFileExportService.cs
@@ -22,7 +22,7 @@ public async Task<(string filePath, bool success, string checkSum, bool isErased
             Directory.CreateDirectory(targetDirectory);
             var filePath = Path.Combine(targetDirectory, $"{DateTime.Now:yyyyMMddhhmmssfff}_{cashboxIdentification.RemoveInvalidFilenameChars()}.tar");
 
-
+            var sha256CheckSum = string.Empty;
             logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section ExportDataAsync [enter].");
             using (var fileStream = File.Create(filePath))
             {
@@ -46,13 +46,15 @@ public async Task<(string filePath, bool success, string checkSum, bool isErased
                         logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Convert.FromBase64String [exit].");
                     }
                 } while (!export.TarFileEndOfFile);
+
+                logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Sha256ChecksumBase64 [enter].");
+                using var sha256 = SHA256.Create();
+                fileStream.Position = 0;
+                sha256CheckSum = Convert.ToBase64String(sha256.ComputeHash(fileStream));
+                logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Sha256ChecksumBase64 [exit].");
             }
             logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section ExportDataAsync [exit].");
 
-            logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Sha256ChecksumBase64 [enter].");
-            using var sha256 = SHA256.Create();
-            var sha256CheckSum = Convert.ToBase64String(sha256.ComputeHash(File.ReadAllBytes(filePath)));
-            logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Sha256ChecksumBase64 [exit].");
             var endSessionRequest = new EndExportSessionRequest
             {
                 TokenId = exportSession.TokenId,
@@ -64,6 +66,7 @@ public async Task<(string filePath, bool success, string checkSum, bool isErased
             {
                 return (null, false, null, false);
             }
+
             logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync [exit].");
             return (filePath, true, sha256CheckSum, endExportSessionResult.IsErased);
         }

diff --git a/...ltrust.Middleware.Localization.QueueDE/fiskaltrust.Middleware.Localization.QueueDE.csproj b/...ltrust.Middleware.Localization.QueueDE/fiskaltrust.Middleware.Localization.QueueDE.csproj
@@ -2,7 +2,7 @@
 
     <ItemGroup>
         <PackageReference Include="fiskaltrust.Exports.DSFinVK" Version="1.6.4" />
-        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.1" />
+        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.2" />
         <PackageReference Include="fiskaltrust.Middleware.Abstractions" Version="1.3.3" />
         <PackageReference Include="fiskaltrust.storage.encryption" Version="1.3.1" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="3.1.4" />

diff --git a/queue/src/fiskaltrust.Middleware.Queue.EF/fiskaltrust.Middleware.Queue.EF.csproj b/queue/src/fiskaltrust.Middleware.Queue.EF/fiskaltrust.Middleware.Queue.EF.csproj
@@ -18,7 +18,7 @@
         <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="1.1.1" />
         <PackageReference Include="System.ValueTuple" Version="4.5.0" />
         <PackageReference Include="fiskaltrust.Exports.DSFinVK" Version="1.6.4" />
-        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.1" />
+        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.2" />
         <PackageReference Include="EntityFramework" Version="6.4.4" />
         <PackageReference Include="SharpCompress" Version="0.28.2" />
         <PackageReference Include="System.Text.Encoding.CodePages" Version="5.0.0" />

diff --git a/queue/src/fiskaltrust.Middleware.Queue.InMemory/fiskaltrust.Middleware.Queue.InMemory.csproj b/queue/src/fiskaltrust.Middleware.Queue.InMemory/fiskaltrust.Middleware.Queue.InMemory.csproj
@@ -18,7 +18,7 @@
         <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="1.1.1" />
         <PackageReference Include="System.ValueTuple" Version="4.5.0" />
         <PackageReference Include="fiskaltrust.Exports.DSFinVK" Version="1.6.4" />
-        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.1" />
+        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.2" />
         <PackageReference Include="SharpCompress" Version="0.28.2" />
         <PackageReference Include="CsvHelper" Version="15.0.5" />
     </ItemGroup>

diff --git a/queue/src/fiskaltrust.Middleware.Queue.MySQL/fiskaltrust.Middleware.Queue.MySQL.csproj b/queue/src/fiskaltrust.Middleware.Queue.MySQL/fiskaltrust.Middleware.Queue.MySQL.csproj
@@ -16,7 +16,7 @@
         <PackageReference Include="Dapper" Version="2.0.35" />
         <PackageReference Include="MySqlConnector" Version="2.1.10" />
         <PackageReference Include="fiskaltrust.Exports.DSFinVK" Version="1.6.4" />
-        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.1" />
+        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.2" />
         <PackageReference Include="SharpCompress" Version="0.28.2" />
         <PackageReference Include="System.Text.Encoding.CodePages" Version="5.0.0" />
     </ItemGroup>

diff --git a/...rc/fiskaltrust.Middleware.Queue.PostgreSQL/fiskaltrust.Middleware.Queue.PostgreSQL.csproj b/...rc/fiskaltrust.Middleware.Queue.PostgreSQL/fiskaltrust.Middleware.Queue.PostgreSQL.csproj
@@ -14,7 +14,7 @@
         <PackageReference Include="Microsoft.EntityFrameworkCore" Version="5.0.10" />
         <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="5.0.10" />
         <PackageReference Include="fiskaltrust.Exports.DSFinVK" Version="1.6.4" />
-        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.1" />
+        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.2" />
         <PackageReference Include="SharpCompress" Version="0.28.2" />
         <PackageReference Include="CsvHelper" Version="15.0.5" />
     </ItemGroup>

diff --git a/queue/src/fiskaltrust.Middleware.Queue.SQLite/.nuspec b/queue/src/fiskaltrust.Middleware.Queue.SQLite/.nuspec
@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<package >
+<package>
   <metadata>
     <id>fiskaltrust.Middleware.Queue.SQLite</id>
     <version>1.3.0-local</version>
@@ -23,11 +23,11 @@
         <dependency id="fiskaltrust.storage" version="1.3.46-rc1" />
         <dependency id="Dapper" version="2.0.123" />
         <dependency id="fiskaltrust.Exports.DSFinVK" version="1.6.4" />
-        <dependency id="fiskaltrust.Exports.TAR" version="1.1.1" />
+        <dependency id="fiskaltrust.Exports.TAR" version="1.1.2" />
         <dependency id="Microsoft.Data.Sqlite" version="3.1.4" />
         <dependency id="System.Linq.Async" version="4.1.1" />
-        <dependency id="SharpCompress" version="0.28.2"/>
-        <dependency id="CsvHelper" version="28.0.1"/>
+        <dependency id="SharpCompress" version="0.28.2" />
+        <dependency id="CsvHelper" version="28.0.1" />
       </group>
     </dependencies>
   </metadata>
@@ -42,29 +42,46 @@
     <file src="bin\Release\netstandard2.0\publish\**\*.config" target="lib\netstandard2.0" />
     <file src="bin\Release\netstandard2.0\publish\**\*.exe" target="lib\netstandard2.0" />
     <file src="bin\Release\netstandard2.0\publish\**\*.dll" target="lib\netstandard2.0" />
-    <file src="bin\Release\netstandard2.0\publish\Migrations\*.*" target="lib\netstandard2.0\Migrations" />
+    <file src="bin\Release\netstandard2.0\publish\Migrations\*.*"
+      target="lib\netstandard2.0\Migrations" />
 
 
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Contracts.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueDE.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueDEFAULT.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueME.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueAT.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueFR.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueIT.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueES.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Queue.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Queue.SQLite.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.QueueSynchronizer.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Storage.Base.dll" target="lib\netstandard2.1" />
-    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Storage.SQLite.dll" target="lib\netstandard2.1" />
-
-    <file src="bin\Release\netstandard2.1\publish\Migrations\*.*" target="lib\netstandard2.1\Migrations" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Contracts.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueDE.dll"
+      target="lib\netstandard2.1" />
+    <file
+      src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueDEFAULT.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueME.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueAT.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueFR.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueIT.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Localization.QueueES.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Queue.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Queue.SQLite.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.QueueSynchronizer.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Storage.Base.dll"
+      target="lib\netstandard2.1" />
+    <file src="bin\Release\netstandard2.1\publish\fiskaltrust.Middleware.Storage.SQLite.dll"
+      target="lib\netstandard2.1" />
+
+    <file src="bin\Release\netstandard2.1\publish\Migrations\*.*"
+      target="lib\netstandard2.1\Migrations" />
 
     <file src="bin\Release\netstandard2.0\runtimes\**\*.*" target="runtimes" />
 
-    <file src="fiskaltrust.Middleware.Queue.SQLite.targets" target="build\fiskaltrust.Middleware.Queue.SQLite.targets" />
+    <file src="fiskaltrust.Middleware.Queue.SQLite.targets"
+      target="build\fiskaltrust.Middleware.Queue.SQLite.targets" />
     <file src="bin\Release\netstandard2.0\publish\Migrations\*.*" target="contentFiles\Migrations" />
     <file src="bin\Release\netstandard2.0\publish\LICENSES\**" target="LICENSES" />
   </files>
-</package>
+</package>
diff --git a/queue/src/fiskaltrust.Middleware.Queue.SQLite/fiskaltrust.Middleware.Queue.SQLite.csproj b/queue/src/fiskaltrust.Middleware.Queue.SQLite/fiskaltrust.Middleware.Queue.SQLite.csproj
@@ -16,7 +16,7 @@
         <PackageReference Include="fiskaltrust.storage.serialization" Version="1.3.1" />
         <PackageReference Include="Dapper" Version="2.0.123" />
         <PackageReference Include="fiskaltrust.Exports.DSFinVK" Version="1.6.4" />
-        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.1" />
+        <PackageReference Include="fiskaltrust.Exports.TAR" Version="1.1.2" />
         <PackageReference Include="System.Linq.Async" Version="4.1.1" />
         <PackageReference Include="SharpCompress" Version="0.28.2" />
         <PackageReference Include="CsvHelper" Version="15.0.5" />

diff --git a/scu-de/src/fiskaltrust.Middleware.SCU.DE.Swissbit/Interop/ISwissbitProxy.cs b/scu-de/src/fiskaltrust.Middleware.SCU.DE.Swissbit/Interop/ISwissbitProxy.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Threading.Tasks;
 using fiskaltrust.ifPOS.v1.de;
 using fiskaltrust.Middleware.SCU.DE.Swissbit.Models;
@@ -34,9 +35,9 @@ public interface ISwissbitProxy : IDisposable
         public Task<TransactionResponse> TransactionUpdateAsync(string clientId, UInt64 transactionNumber, byte[] processData, string processType);
         public Task<TransactionResponse> TransactionFinishAsync(string clientId, UInt64 transactionNumber, byte[] processData, string processType);
         public Task<List<ulong>> GetStartedTransactionsAsync(string clientId);
-        public Task ExportTarAsync(System.IO.Stream stream);
-        public Task ExportTarFilteredTimeAsync(System.IO.Stream stream, UInt64 startDateUnixTime, UInt64 endDateUnixTime, string clientId);
-        public Task ExportTarFilteredTransactionAsync(System.IO.Stream stream, UInt64 startTransactionNumber, UInt64 endTransactionNumber, string clientId);
+        public Task ExportTarAsync(Stream stream);
+        public Task ExportTarFilteredTimeAsync(Stream stream, UInt64 startDateUnixTime, UInt64 endDateUnixTime, string clientId);
+        public Task ExportTarFilteredTransactionAsync(Stream stream, UInt64 startTransactionNumber, UInt64 endTransactionNumber, string clientId);
         public Task<byte[]> GetLogMessageCertificateAsync();
         public Task DeleteStoredDataAsync();
     }