Initial commit
This commit is contained in:
6
azala.info Scraper/App.config
Normal file
6
azala.info Scraper/App.config
Normal file
@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8" ?>
|
||||
<configuration>
|
||||
<startup>
|
||||
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
|
||||
</startup>
|
||||
</configuration>
|
||||
16
azala.info Scraper/Card.cs
Normal file
16
azala.info Scraper/Card.cs
Normal file
@ -0,0 +1,16 @@
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace azala.info_Scraper {
|
||||
internal class Card {
|
||||
[JsonProperty(PropertyName = "id")]
|
||||
public long ID { get; set; }
|
||||
[JsonProperty(PropertyName = "card_text")]
|
||||
public string Text { get; set; }
|
||||
[JsonProperty(PropertyName = "num_blanks")]
|
||||
public int Blanks { get; set; }
|
||||
[JsonProperty(PropertyName = "o")]
|
||||
public int Order { get; set; }
|
||||
[JsonProperty(PropertyName = "deck_id")]
|
||||
public int DeckID { get; internal set; }
|
||||
}
|
||||
}
|
||||
174
azala.info Scraper/Program.cs
Normal file
174
azala.info Scraper/Program.cs
Normal file
@ -0,0 +1,174 @@
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using MySql.Data;
|
||||
using MySql.Data.MySqlClient;
|
||||
using System.Data;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
|
||||
namespace azala.info_Scraper {
|
||||
public class Program {
|
||||
private readonly MySqlConnection connection;
|
||||
|
||||
public Program() {
|
||||
connection = new MySqlConnection($"Server={"localhost"};Database={"cardsagainsthumanity"};Uid={"dbuser"};Pwd={"DSKJjojofsa9%=(!=i2100"};");
|
||||
connection.StateChange += Connection_StateChange;
|
||||
errorstream = new StreamWriter("errors.txt", true);
|
||||
}
|
||||
|
||||
private void Connection_StateChange(object sender, StateChangeEventArgs e) {
|
||||
Console.WriteLine("MySQL connection is now {0}!", e.CurrentState);
|
||||
}
|
||||
|
||||
public async Task OpenConnection() {
|
||||
Console.WriteLine("Connecting to MySQL Server...");
|
||||
await connection.OpenAsync();
|
||||
}
|
||||
~Program() {
|
||||
connection.Close();
|
||||
}
|
||||
|
||||
public static async Task Main() {
|
||||
var program = new Program();
|
||||
var openTask = program.OpenConnection();
|
||||
Console.WriteLine("Give a directory:");
|
||||
var path = XConsole.ReadDirectory();
|
||||
try {
|
||||
await openTask;
|
||||
program.Run(path);
|
||||
#pragma warning disable CA1031 // Do not catch general exception types
|
||||
} catch (Exception e) {
|
||||
Console.WriteLine("Uncaught ERROR: {0}", e);
|
||||
}
|
||||
#pragma warning restore CA1031 // Do not catch general exception types
|
||||
Console.WriteLine("Program end...");
|
||||
Console.ReadLine();
|
||||
}
|
||||
|
||||
private static readonly string[] searchPatterns = { "*.que", "*.ans" };
|
||||
private readonly StreamWriter errorstream;
|
||||
private string processeddirectory;
|
||||
|
||||
private void Run(string path) {
|
||||
processeddirectory = Path.Combine(path, "processed");
|
||||
if (connection.State != ConnectionState.Open)
|
||||
throw new Exception("Connection not opened!");
|
||||
Console.WriteLine("Creating tables...");
|
||||
try {
|
||||
CreateTables();
|
||||
} catch (MySqlException e) {
|
||||
Console.WriteLine("ERROR: Creating tables: {0}", e.Message);
|
||||
throw;
|
||||
}
|
||||
Console.WriteLine("Tables created!");
|
||||
while (true) {
|
||||
foreach (var file in XDirectory.GetFiles(path, searchPatterns, SearchOption.TopDirectoryOnly)) {
|
||||
try {
|
||||
ProcessFile(file);
|
||||
Console.WriteLine();
|
||||
} catch (MySqlException e) {
|
||||
Console.WriteLine("MySQL ERROR: Processing file failed: {0}", e.Message);
|
||||
} catch (ArgumentException e) {
|
||||
Console.WriteLine("Argument exception: {0}", e.Message);
|
||||
}
|
||||
}
|
||||
Console.WriteLine("Waiting 1s..");
|
||||
Thread.Sleep(1000);
|
||||
}
|
||||
}
|
||||
|
||||
private void CreateTables() {
|
||||
CreateCardsTable();
|
||||
}
|
||||
|
||||
private void CreateCardsTable() {
|
||||
var command = connection.CreateCommand();
|
||||
command.CommandText = "CREATE TABLE IF NOT EXISTS cards(id INTEGER NOT NULL PRIMARY KEY,deck_id INTEGER NOT NULL,card_text VARCHAR(2048) NOT NULL,num_blanks INTEGER NOT NULL,o INTEGER NOT NULL);";
|
||||
if (command.ExecuteNonQuery() == 1)
|
||||
Console.WriteLine("Table created!");
|
||||
else
|
||||
Console.WriteLine("Table not changed!");
|
||||
}
|
||||
|
||||
private void ProcessFile(string file) {
|
||||
var filename = Path.GetFileName(file);
|
||||
Console.WriteLine("Processing file '{0}'", filename);
|
||||
var file_id = Path.GetFileNameWithoutExtension(filename);
|
||||
|
||||
if (!Directory.Exists(processeddirectory))
|
||||
Directory.CreateDirectory(processeddirectory);
|
||||
|
||||
Console.WriteLine("Id is {0}", file_id);
|
||||
|
||||
if (int.TryParse(file_id, out var deck_id)) {
|
||||
var contents = File.ReadAllText(file);
|
||||
var cardlist = JsonConvert.DeserializeObject<List<Card>>(contents);
|
||||
var insertedIds = new HashSet<long>();
|
||||
|
||||
foreach (var card in cardlist) {
|
||||
if (insertedIds.Contains(card.ID)) {
|
||||
Console.WriteLine("Card with id '{0}' already inserted! Skipping...", card.ID);
|
||||
continue;
|
||||
}
|
||||
card.DeckID = deck_id;
|
||||
AddCard(card);
|
||||
insertedIds.Add(card.ID);
|
||||
}
|
||||
Console.WriteLine("Sucessfully inserted {0}/{1} new cards!", insertedIds.Count, cardlist.Count);
|
||||
File.Move(file, Path.Combine(processeddirectory, filename));
|
||||
} else {
|
||||
throw new ArgumentException($"File '{filename}' with non-integer ID!");
|
||||
}
|
||||
}
|
||||
|
||||
private void AddCard(Card card) {
|
||||
using var command = connection.CreateCommand();
|
||||
command.CommandText = "INSERT IGNORE INTO cards(id,deck_id,card_text,num_blanks,o) VALUES (@id,@deckid,@text,@blanks,@order);";
|
||||
command.Parameters.AddWithValue("@deckid", card.DeckID);
|
||||
command.Parameters.AddWithValue("@id", card.ID);
|
||||
command.Parameters.AddWithValue("@text", card.Text);
|
||||
command.Parameters.AddWithValue("@blanks", card.Blanks);
|
||||
command.Parameters.AddWithValue("@order", card.Order);
|
||||
var result = command.ExecuteNonQuery();
|
||||
if (result != 1) {
|
||||
Console.WriteLine("ERROR: {0} lines affected!", result);
|
||||
errorstream.WriteLine("[{0}] Insertion: {1} rows affected!\n{2}", DateTime.Now.ToString(), result, JsonConvert.SerializeObject(card));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal class XConsole {
|
||||
internal static string ReadDirectory() {
|
||||
while (true) {
|
||||
var input = Console.ReadLine();
|
||||
if (Directory.Exists(input))
|
||||
return input;
|
||||
Console.WriteLine("Directory doesn't exist!");
|
||||
}
|
||||
}
|
||||
}
|
||||
public static class XDirectory { // Regex version
|
||||
public static IEnumerable<string> GetFiles(string path,
|
||||
string searchPatternExpression = "",
|
||||
SearchOption searchOption = SearchOption.TopDirectoryOnly) {
|
||||
var reSearchPattern = new Regex(searchPatternExpression, RegexOptions.IgnoreCase);
|
||||
return Directory.EnumerateFiles(path, "*", searchOption)
|
||||
.Where(file =>
|
||||
reSearchPattern.IsMatch(Path.GetExtension(file)));
|
||||
}
|
||||
|
||||
// Takes same patterns, and executes in parallel
|
||||
public static IEnumerable<string> GetFiles(string path,
|
||||
string[] searchPatterns,
|
||||
SearchOption searchOption = SearchOption.TopDirectoryOnly) {
|
||||
return searchPatterns.AsParallel()
|
||||
.SelectMany(searchPattern =>
|
||||
Directory.EnumerateFiles(path, searchPattern, searchOption));
|
||||
}
|
||||
}
|
||||
}
|
||||
56
azala.info Scraper/Program.cs.old
Normal file
56
azala.info Scraper/Program.cs.old
Normal file
@ -0,0 +1,56 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.WebSockets;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Websocket.Client;
|
||||
|
||||
namespace azala.info_Scraper {
|
||||
class Program {
|
||||
private const string AZALA_INFO_URL = "wss://azala.info/main";
|
||||
private readonly WebsocketClient socket;
|
||||
|
||||
public Program(string url) {
|
||||
Console.WriteLine("Connecting to {0}", url);
|
||||
var uri = new Uri(url);
|
||||
|
||||
socket = new WebsocketClient(uri) {
|
||||
ReconnectTimeoutMs = (int)TimeSpan.FromSeconds(30).TotalMilliseconds
|
||||
};
|
||||
socket.ReconnectionHappened.Subscribe(type =>
|
||||
Console.WriteLine($"Reconnection happened, type: {type}"));
|
||||
socket.MessageReceived.Subscribe(MessageReceived);
|
||||
}
|
||||
|
||||
private void MessageReceived(ResponseMessage msg) {
|
||||
if (msg.MessageType == WebSocketMessageType.Text) {
|
||||
TextMessage(msg.Text);
|
||||
}
|
||||
}
|
||||
|
||||
private void TextMessage(string text) {
|
||||
try {
|
||||
|
||||
} catch (Exception e) {
|
||||
Console.WriteLine("Cannot parse message (JSON expected): {0}",e);
|
||||
}
|
||||
}
|
||||
|
||||
static async Task Main() {
|
||||
try {
|
||||
await new Program(AZALA_INFO_URL).Run();
|
||||
#pragma warning disable CA1031 // Do not catch general exception types
|
||||
} catch (Exception e) {
|
||||
Console.WriteLine("Uncaught error:\n{0}", e);
|
||||
}
|
||||
#pragma warning restore CA1031 // Do not catch general exception types
|
||||
}
|
||||
|
||||
private async Task Run() {
|
||||
await socket.Start();
|
||||
Console.ReadLine();
|
||||
}
|
||||
}
|
||||
}
|
||||
36
azala.info Scraper/Properties/AssemblyInfo.cs
Normal file
36
azala.info Scraper/Properties/AssemblyInfo.cs
Normal file
@ -0,0 +1,36 @@
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
// Allgemeine Informationen über eine Assembly werden über die folgenden
|
||||
// Attribute gesteuert. Ändern Sie diese Attributwerte, um die Informationen zu ändern,
|
||||
// die einer Assembly zugeordnet sind.
|
||||
[assembly: AssemblyTitle("azala.info Scraper")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("azala.info Scraper")]
|
||||
[assembly: AssemblyCopyright("Copyright © 2019")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Durch Festlegen von ComVisible auf FALSE werden die Typen in dieser Assembly
|
||||
// für COM-Komponenten unsichtbar. Wenn Sie auf einen Typ in dieser Assembly von
|
||||
// COM aus zugreifen müssen, sollten Sie das ComVisible-Attribut für diesen Typ auf "True" festlegen.
|
||||
[assembly: ComVisible(false)]
|
||||
|
||||
// Die folgende GUID bestimmt die ID der Typbibliothek, wenn dieses Projekt für COM verfügbar gemacht wird
|
||||
[assembly: Guid("886207e9-c764-47f1-a5e5-1556a9e28648")]
|
||||
|
||||
// Versionsinformationen für eine Assembly bestehen aus den folgenden vier Werten:
|
||||
//
|
||||
// Hauptversion
|
||||
// Nebenversion
|
||||
// Buildnummer
|
||||
// Revision
|
||||
//
|
||||
// Sie können alle Werte angeben oder Standardwerte für die Build- und Revisionsnummern verwenden,
|
||||
// indem Sie "*" wie unten gezeigt eingeben:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
90
azala.info Scraper/azala.info Scraper.csproj
Normal file
90
azala.info Scraper/azala.info Scraper.csproj
Normal file
@ -0,0 +1,90 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{886207E9-C764-47F1-A5E5-1556A9E28648}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>azala.info_Scraper</RootNamespace>
|
||||
<AssemblyName>azala.info Scraper</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
||||
<Deterministic>true</Deterministic>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<LangVersion>8.0</LangVersion>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<LangVersion>8.0</LangVersion>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="BouncyCastle.Crypto, Version=1.8.3.0, Culture=neutral, PublicKeyToken=0e99375e54769942">
|
||||
<HintPath>..\packages\BouncyCastle.1.8.3.1\lib\BouncyCastle.Crypto.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Google.Protobuf, Version=3.6.1.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Google.Protobuf.3.6.1\lib\net45\Google.Protobuf.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="MySql.Data, Version=8.0.18.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\MySql.Data.8.0.18\lib\net452\MySql.Data.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Renci.SshNet, Version=2016.1.0.0, Culture=neutral, PublicKeyToken=1cee9f8bde3db106, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\SSH.NET.2016.1.0\lib\net40\Renci.SshNet.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.ComponentModel" />
|
||||
<Reference Include="System.ComponentModel.DataAnnotations" />
|
||||
<Reference Include="System.Configuration" />
|
||||
<Reference Include="System.Configuration.Install" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Drawing" />
|
||||
<Reference Include="System.Drawing.Design" />
|
||||
<Reference Include="System.Management" />
|
||||
<Reference Include="System.Reactive, Version=4.0.0.0, Culture=neutral, PublicKeyToken=94bc3704cddfc263, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\System.Reactive.4.0.0\lib\net46\System.Reactive.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Transactions" />
|
||||
<Reference Include="System.Windows" />
|
||||
<Reference Include="System.Windows.Forms" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="Microsoft.CSharp" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Xml" />
|
||||
<Reference Include="Websocket.Client, Version=3.2.0.0, Culture=neutral, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Websocket.Client.3.2.56\lib\netstandard2.0\Websocket.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="WindowsBase" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="Program.cs.old" />
|
||||
<Compile Include="Card.cs" />
|
||||
<Compile Include="Program.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config" />
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
</Project>
|
||||
10
azala.info Scraper/packages.config
Normal file
10
azala.info Scraper/packages.config
Normal file
@ -0,0 +1,10 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="BouncyCastle" version="1.8.3.1" targetFramework="net472" />
|
||||
<package id="Google.Protobuf" version="3.6.1" targetFramework="net472" />
|
||||
<package id="MySql.Data" version="8.0.18" targetFramework="net472" />
|
||||
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
|
||||
<package id="SSH.NET" version="2016.1.0" targetFramework="net472" />
|
||||
<package id="System.Reactive" version="4.0.0" targetFramework="net472" />
|
||||
<package id="Websocket.Client" version="3.2.56" targetFramework="net472" />
|
||||
</packages>
|
||||
Reference in New Issue
Block a user