Created
December 7, 2017 08:27
-
-
Save keimpema/ec962384d5fe3eb7a5f5030353ba9e2b to your computer and use it in GitHub Desktop.
A decompresing INntpConnection implementation (not very good)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
using System; | |
using System.Collections.Generic; | |
using System.IO; | |
using System.Net.Security; | |
using System.Net.Sockets; | |
using System.Threading; | |
using System.Threading.Tasks; | |
using ICSharpCode.SharpZipLib.Zip.Compression; | |
using ICSharpCode.SharpZipLib.Zip.Compression.Streams; | |
using Microsoft.Extensions.Logging; | |
using Usenet.Exceptions; | |
using Usenet.Extensions; | |
using Usenet.Nntp.Parsers; | |
using Usenet.Nntp.Responses; | |
using Usenet.Util; | |
using Usenet.Yenc; | |
namespace Usenet.Nntp | |
{ | |
public class NntpDecompressingConnection : INntpConnection | |
{ | |
private static readonly ILogger log = Logger.Create<NntpDecompressingConnection>(); | |
private const int bufferSize = 4096; // 8192 | |
private const int yencLineBufferSize = 1024; | |
private const int readTimeout = 500; // msec | |
private const int lfByte = 10; | |
private const int crByte = 13; | |
private const int spaceByte = 32; | |
private const int dotByte = 46; | |
private const int zeroByte = 48; | |
private const int nineByte = 57; | |
private static readonly byte[] yBeginBytes = UsenetEncoding.Default.GetBytes(YencKeywords.Header); | |
private static readonly byte[] yEndBytes = UsenetEncoding.Default.GetBytes(YencKeywords.Footer); | |
private readonly byte[] buffer = new byte[bufferSize]; | |
private readonly byte[] yencLineBuffer = new byte[yencLineBufferSize]; | |
private readonly TcpClient client = new TcpClient(); | |
private readonly MemoryStream readBufferStream = new MemoryStream(); | |
private readonly MemoryStream copyStream = new MemoryStream(); | |
private readonly MemoryStream yencStream = new MemoryStream(); | |
private Stream nntpStream; | |
private StreamWriter writer; | |
public async Task<TResponse> ConnectAsync<TResponse>(string hostname, int port, bool useSsl, IResponseParser<TResponse> parser) | |
{ | |
log.LogInformation("Connecting: {hostname} {port} (Use SSl = {useSsl})", hostname, port, useSsl); | |
await client.ConnectAsync(hostname, port); | |
nntpStream = await GetStreamAsync(hostname, useSsl); | |
writer = new StreamWriter(nntpStream, UsenetEncoding.Default) { AutoFlush = true }; | |
return GetResponse(parser); | |
} | |
public TResponse Command<TResponse>(string command, IResponseParser<TResponse> parser) | |
{ | |
ThrowIfNotConnected(); | |
log.LogInformation("Sending command: {Command}", | |
command.StartsWith("AUTHINFO PASS", StringComparison.Ordinal) ? "AUTHINFO PASS [omitted]" : command); | |
// send command and get response | |
writer.WriteLine(command); | |
return GetResponse(parser); | |
} | |
public TResponse MultiLineCommand<TResponse>(string command, IMultiLineResponseParser<TResponse> parser) | |
{ | |
return MultiLineCommand(command, false, parser); | |
} | |
public TResponse DecompressingMultiLineCommand<TResponse>(string command, IMultiLineResponseParser<TResponse> parser) | |
{ | |
return MultiLineCommand(command, true, parser); | |
} | |
private TResponse MultiLineCommand<TResponse>(string command, bool mustDecompress, IMultiLineResponseParser<TResponse> parser) | |
{ | |
// send command | |
NntpResponse response = Command(command, new ResponseParser()); | |
mustDecompress = mustDecompress || response.Message.Contains("COMPRESS", StringComparison.OrdinalIgnoreCase); | |
// receive multiline datablock (decompress stream if needed) | |
List<string> dataBlock = response.Success | |
? ReadMultiLineDataBlock(mustDecompress ? Decompress(nntpStream) : nntpStream) | |
: EmptyList<string>.Instance; | |
// create response object | |
return parser.Parse(response.Code, response.Message, dataBlock); | |
} | |
public TResponse GetResponse<TResponse>(IResponseParser<TResponse> parser) | |
{ | |
string responseText = ReadLine(nntpStream, true); | |
log.LogInformation("Response received: {Response}", responseText); | |
if (responseText == null) | |
{ | |
throw new NntpException("Received no response."); | |
} | |
if (responseText.Length < 3 || !int.TryParse(responseText.Substring(0, 3), out int code)) | |
{ | |
throw new NntpException("Received invalid response."); | |
} | |
return parser.Parse(code, responseText.Substring(4)); | |
} | |
public void WriteLine(string line) | |
{ | |
ThrowIfNotConnected(); | |
writer.WriteLine(line); | |
} | |
private void ThrowIfNotConnected() | |
{ | |
if (!client.Connected) | |
{ | |
throw new NntpException("Client not connected."); | |
} | |
} | |
/// <summary> | |
/// Read line from binary stream. | |
/// Source: https://stackoverflow.com/questions/18358435/how-can-i-use-the-deflatestream-class-on-one-line-in-a-file | |
/// </summary> | |
/// <param name="source"></param> | |
/// <param name="mustSkipGarbage"></param> | |
/// <returns></returns> | |
private string ReadLine(Stream source, bool mustSkipGarbage) | |
{ | |
readBufferStream.SetLength(0); | |
int next; | |
var offset = 0; | |
var foundPrefix = false; | |
var wasCr = false; | |
var garbageCount = 0; | |
var digitCount = 0; | |
while ((next = source.ReadByte()) >= 0) | |
{ | |
if (next == lfByte && wasCr) | |
{ | |
// CRLF found | |
// end of line (minus the CR) | |
return UsenetEncoding.Default.GetString(readBufferStream.ToArray(), offset, (int)readBufferStream.Length - offset -1); | |
} | |
if (mustSkipGarbage && !foundPrefix) | |
{ | |
if (next == spaceByte && digitCount >= 3) | |
{ | |
// found starting prefix: a 3 digit code + 1 space | |
offset = (int) readBufferStream.Length - 3; | |
foundPrefix = true; | |
if (garbageCount > 3) | |
{ | |
log.LogWarning("Skipped garbage: {GarbageCount} bytes", garbageCount - 3); | |
} | |
} | |
else if (next < zeroByte || next > nineByte) | |
{ | |
digitCount = 0; | |
} | |
else | |
{ | |
digitCount++; | |
} | |
garbageCount++; | |
} | |
readBufferStream.WriteByte((byte)next); | |
wasCr = next == crByte; | |
} | |
// end of file | |
return readBufferStream.Length == 0 | |
? null | |
: UsenetEncoding.Default.GetString(readBufferStream.ToArray(), offset, (int) readBufferStream.Length - offset); | |
} | |
private static int ReadLineBytes(Stream source, byte[] lineBuffer) | |
{ | |
int next; | |
int lineBufferSize = lineBuffer.Length; | |
var lineIndex = 0; | |
var wasCr = false; | |
while ((next = source.ReadByte()) >= 0) | |
{ | |
if (next == lfByte && wasCr) | |
{ | |
// CRLF found | |
// end of line (minus the CR) | |
return lineIndex - 1; | |
} | |
if (lineIndex >= lineBufferSize) | |
{ | |
// buffer full | |
return lineBufferSize; | |
} | |
lineBuffer[lineIndex++] = (byte)next; | |
wasCr = next == crByte; | |
} | |
// end of file | |
return lineIndex; | |
} | |
private async Task<Stream> GetStreamAsync(string hostname, bool useSsl) | |
{ | |
NetworkStream stream = client.GetStream(); | |
if (!useSsl) | |
{ | |
return stream; | |
} | |
var sslStream = new SslStream(stream); | |
await sslStream.AuthenticateAsClientAsync(hostname); | |
return sslStream; | |
} | |
private List<string> ReadMultiLineDataBlock(Stream readStream) | |
{ | |
var lines = new List<string>(); | |
string line; | |
while ((line = ReadLine(readStream, false)) != null) | |
{ | |
if (line == ".") | |
{ | |
break; | |
} | |
lines.Add(line.StartsWith("..") ? line.Substring(1) : line); | |
} | |
return lines; | |
} | |
private Stream Decompress(Stream source) | |
{ | |
// reset buffer | |
copyStream.SetLength(0); | |
// cannot stream but need to copy entire data block into memory because | |
// protocol does not work with a termintator | |
var byteCount = (int) CopyBinaryData(source, copyStream); | |
if (byteCount == 0) | |
{ | |
return null; | |
} | |
// astraweb uses yenc compression | |
// source: http://helpdesk.astraweb.com/index.php?_m=news&_a=viewnews&newsid=9 | |
// just deflate if no yenc encoding is used | |
copyStream.Position = 0; | |
copyStream.Read(buffer, 0, yBeginBytes.Length); | |
copyStream.Position = 0; | |
if (!buffer.StartsWith(yBeginBytes)) | |
{ | |
return new InflaterInputStream(copyStream, new Inflater(false), byteCount); | |
} | |
// skip yenc header line | |
ReadLineBytes(copyStream, yencLineBuffer); | |
// reset yenc buffer | |
yencStream.SetLength(0); | |
while ((byteCount = ReadLineBytes(copyStream, yencLineBuffer)) > 0) | |
{ | |
//log.LogDebug(UsenetEncoding.Default.GetString(lineBuffer, 0, byteCount)); | |
var offset = 0; | |
if (yencLineBuffer[0] == dotByte) | |
{ | |
if (byteCount == 1) | |
{ | |
break; | |
} | |
if (yencLineBuffer[1] == dotByte) | |
{ | |
offset = 1; | |
} | |
} | |
else | |
{ | |
if (yencLineBuffer.StartsWith(yEndBytes)) | |
{ | |
break; | |
} | |
} | |
int decodedCount = YencLineDecoder.Decode(yencLineBuffer, offset, byteCount - offset, buffer, 0); | |
yencStream.Write(buffer, 0, decodedCount); | |
} | |
// deflate | |
yencStream.Position = 0; | |
return new InflaterInputStream(yencStream, new Inflater(true), (int)yencStream.Length); | |
} | |
private long CopyBinaryData(Stream source, Stream target) | |
{ | |
try | |
{ | |
int byteCount; | |
source.ReadTimeout = readTimeout; | |
while ((byteCount = source.Read(buffer, 0, bufferSize)) > 0) | |
{ | |
target.Write(buffer, 0, byteCount); | |
} | |
return target.Length; | |
} | |
catch (IOException) | |
{ | |
// swallow | |
return target.Length; | |
} | |
finally | |
{ | |
source.ReadTimeout = Timeout.Infinite; | |
} | |
} | |
public void Dispose() | |
{ | |
client?.Dispose(); | |
writer?.Dispose(); | |
readBufferStream?.Dispose(); | |
nntpStream?.Dispose(); | |
} | |
} | |
} | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment