Initial commit
This commit is contained in:
commit
0edcbdca10
GPG Key ID:
5845BF3F82D5F629
398
.gitignore
vendored
Normal file
398
.gitignore
vendored
Normal file
@ -0,0 +1,398 @@
|
|||||||
|
## Ignore Visual Studio temporary files, build results, and
|
||||||
|
## files generated by popular Visual Studio add-ons.
|
||||||
|
##
|
||||||
|
## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
|
||||||
|
|
||||||
|
# User-specific files
|
||||||
|
*.rsuser
|
||||||
|
*.suo
|
||||||
|
*.user
|
||||||
|
*.userosscache
|
||||||
|
*.sln.docstates
|
||||||
|
|
||||||
|
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||||
|
*.userprefs
|
||||||
|
|
||||||
|
# Mono auto generated files
|
||||||
|
mono_crash.*
|
||||||
|
|
||||||
|
# Build results
|
||||||
|
[Dd]ebug/
|
||||||
|
[Dd]ebugPublic/
|
||||||
|
[Rr]elease/
|
||||||
|
[Rr]eleases/
|
||||||
|
x64/
|
||||||
|
x86/
|
||||||
|
[Ww][Ii][Nn]32/
|
||||||
|
[Aa][Rr][Mm]/
|
||||||
|
[Aa][Rr][Mm]64/
|
||||||
|
bld/
|
||||||
|
[Bb]in/
|
||||||
|
[Oo]bj/
|
||||||
|
[Ll]og/
|
||||||
|
[Ll]ogs/
|
||||||
|
|
||||||
|
# Visual Studio 2015/2017 cache/options directory
|
||||||
|
.vs/
|
||||||
|
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||||
|
#wwwroot/
|
||||||
|
|
||||||
|
# Visual Studio 2017 auto generated files
|
||||||
|
Generated\ Files/
|
||||||
|
|
||||||
|
# MSTest test Results
|
||||||
|
[Tt]est[Rr]esult*/
|
||||||
|
[Bb]uild[Ll]og.*
|
||||||
|
|
||||||
|
# NUnit
|
||||||
|
*.VisualState.xml
|
||||||
|
TestResult.xml
|
||||||
|
nunit-*.xml
|
||||||
|
|
||||||
|
# Build Results of an ATL Project
|
||||||
|
[Dd]ebugPS/
|
||||||
|
[Rr]eleasePS/
|
||||||
|
dlldata.c
|
||||||
|
|
||||||
|
# Benchmark Results
|
||||||
|
BenchmarkDotNet.Artifacts/
|
||||||
|
|
||||||
|
# .NET Core
|
||||||
|
project.lock.json
|
||||||
|
project.fragment.lock.json
|
||||||
|
artifacts/
|
||||||
|
|
||||||
|
# ASP.NET Scaffolding
|
||||||
|
ScaffoldingReadMe.txt
|
||||||
|
|
||||||
|
# StyleCop
|
||||||
|
StyleCopReport.xml
|
||||||
|
|
||||||
|
# Files built by Visual Studio
|
||||||
|
*_i.c
|
||||||
|
*_p.c
|
||||||
|
*_h.h
|
||||||
|
*.ilk
|
||||||
|
*.meta
|
||||||
|
*.obj
|
||||||
|
*.iobj
|
||||||
|
*.pch
|
||||||
|
*.pdb
|
||||||
|
*.ipdb
|
||||||
|
*.pgc
|
||||||
|
*.pgd
|
||||||
|
*.rsp
|
||||||
|
*.sbr
|
||||||
|
*.tlb
|
||||||
|
*.tli
|
||||||
|
*.tlh
|
||||||
|
*.tmp
|
||||||
|
*.tmp_proj
|
||||||
|
*_wpftmp.csproj
|
||||||
|
*.log
|
||||||
|
*.tlog
|
||||||
|
*.vspscc
|
||||||
|
*.vssscc
|
||||||
|
.builds
|
||||||
|
*.pidb
|
||||||
|
*.svclog
|
||||||
|
*.scc
|
||||||
|
|
||||||
|
# Chutzpah Test files
|
||||||
|
_Chutzpah*
|
||||||
|
|
||||||
|
# Visual C++ cache files
|
||||||
|
ipch/
|
||||||
|
*.aps
|
||||||
|
*.ncb
|
||||||
|
*.opendb
|
||||||
|
*.opensdf
|
||||||
|
*.sdf
|
||||||
|
*.cachefile
|
||||||
|
*.VC.db
|
||||||
|
*.VC.VC.opendb
|
||||||
|
|
||||||
|
# Visual Studio profiler
|
||||||
|
*.psess
|
||||||
|
*.vsp
|
||||||
|
*.vspx
|
||||||
|
*.sap
|
||||||
|
|
||||||
|
# Visual Studio Trace Files
|
||||||
|
*.e2e
|
||||||
|
|
||||||
|
# TFS 2012 Local Workspace
|
||||||
|
$tf/
|
||||||
|
|
||||||
|
# Guidance Automation Toolkit
|
||||||
|
*.gpState
|
||||||
|
|
||||||
|
# ReSharper is a .NET coding add-in
|
||||||
|
_ReSharper*/
|
||||||
|
*.[Rr]e[Ss]harper
|
||||||
|
*.DotSettings.user
|
||||||
|
|
||||||
|
# TeamCity is a build add-in
|
||||||
|
_TeamCity*
|
||||||
|
|
||||||
|
# DotCover is a Code Coverage Tool
|
||||||
|
*.dotCover
|
||||||
|
|
||||||
|
# AxoCover is a Code Coverage Tool
|
||||||
|
.axoCover/*
|
||||||
|
!.axoCover/settings.json
|
||||||
|
|
||||||
|
# Coverlet is a free, cross platform Code Coverage Tool
|
||||||
|
coverage*.json
|
||||||
|
coverage*.xml
|
||||||
|
coverage*.info
|
||||||
|
|
||||||
|
# Visual Studio code coverage results
|
||||||
|
*.coverage
|
||||||
|
*.coveragexml
|
||||||
|
|
||||||
|
# NCrunch
|
||||||
|
_NCrunch_*
|
||||||
|
.*crunch*.local.xml
|
||||||
|
nCrunchTemp_*
|
||||||
|
|
||||||
|
# MightyMoose
|
||||||
|
*.mm.*
|
||||||
|
AutoTest.Net/
|
||||||
|
|
||||||
|
# Web workbench (sass)
|
||||||
|
.sass-cache/
|
||||||
|
|
||||||
|
# Installshield output folder
|
||||||
|
[Ee]xpress/
|
||||||
|
|
||||||
|
# DocProject is a documentation generator add-in
|
||||||
|
DocProject/buildhelp/
|
||||||
|
DocProject/Help/*.HxT
|
||||||
|
DocProject/Help/*.HxC
|
||||||
|
DocProject/Help/*.hhc
|
||||||
|
DocProject/Help/*.hhk
|
||||||
|
DocProject/Help/*.hhp
|
||||||
|
DocProject/Help/Html2
|
||||||
|
DocProject/Help/html
|
||||||
|
|
||||||
|
# Click-Once directory
|
||||||
|
publish/
|
||||||
|
|
||||||
|
# Publish Web Output
|
||||||
|
*.[Pp]ublish.xml
|
||||||
|
*.azurePubxml
|
||||||
|
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||||
|
# but database connection strings (with potential passwords) will be unencrypted
|
||||||
|
*.pubxml
|
||||||
|
*.publishproj
|
||||||
|
|
||||||
|
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||||
|
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||||
|
# in these scripts will be unencrypted
|
||||||
|
PublishScripts/
|
||||||
|
|
||||||
|
# NuGet Packages
|
||||||
|
*.nupkg
|
||||||
|
# NuGet Symbol Packages
|
||||||
|
*.snupkg
|
||||||
|
# The packages folder can be ignored because of Package Restore
|
||||||
|
**/[Pp]ackages/*
|
||||||
|
# except build/, which is used as an MSBuild target.
|
||||||
|
!**/[Pp]ackages/build/
|
||||||
|
# Uncomment if necessary however generally it will be regenerated when needed
|
||||||
|
#!**/[Pp]ackages/repositories.config
|
||||||
|
# NuGet v3's project.json files produces more ignorable files
|
||||||
|
*.nuget.props
|
||||||
|
*.nuget.targets
|
||||||
|
|
||||||
|
# Microsoft Azure Build Output
|
||||||
|
csx/
|
||||||
|
*.build.csdef
|
||||||
|
|
||||||
|
# Microsoft Azure Emulator
|
||||||
|
ecf/
|
||||||
|
rcf/
|
||||||
|
|
||||||
|
# Windows Store app package directories and files
|
||||||
|
AppPackages/
|
||||||
|
BundleArtifacts/
|
||||||
|
Package.StoreAssociation.xml
|
||||||
|
_pkginfo.txt
|
||||||
|
*.appx
|
||||||
|
*.appxbundle
|
||||||
|
*.appxupload
|
||||||
|
|
||||||
|
# Visual Studio cache files
|
||||||
|
# files ending in .cache can be ignored
|
||||||
|
*.[Cc]ache
|
||||||
|
# but keep track of directories ending in .cache
|
||||||
|
!?*.[Cc]ache/
|
||||||
|
|
||||||
|
# Others
|
||||||
|
ClientBin/
|
||||||
|
~$*
|
||||||
|
*~
|
||||||
|
*.dbmdl
|
||||||
|
*.dbproj.schemaview
|
||||||
|
*.jfm
|
||||||
|
*.pfx
|
||||||
|
*.publishsettings
|
||||||
|
orleans.codegen.cs
|
||||||
|
|
||||||
|
# Including strong name files can present a security risk
|
||||||
|
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||||
|
#*.snk
|
||||||
|
|
||||||
|
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||||
|
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||||
|
#bower_components/
|
||||||
|
|
||||||
|
# RIA/Silverlight projects
|
||||||
|
Generated_Code/
|
||||||
|
|
||||||
|
# Backup & report files from converting an old project file
|
||||||
|
# to a newer Visual Studio version. Backup files are not needed,
|
||||||
|
# because we have git ;-)
|
||||||
|
_UpgradeReport_Files/
|
||||||
|
Backup*/
|
||||||
|
UpgradeLog*.XML
|
||||||
|
UpgradeLog*.htm
|
||||||
|
ServiceFabricBackup/
|
||||||
|
*.rptproj.bak
|
||||||
|
|
||||||
|
# SQL Server files
|
||||||
|
*.mdf
|
||||||
|
*.ldf
|
||||||
|
*.ndf
|
||||||
|
|
||||||
|
# Business Intelligence projects
|
||||||
|
*.rdl.data
|
||||||
|
*.bim.layout
|
||||||
|
*.bim_*.settings
|
||||||
|
*.rptproj.rsuser
|
||||||
|
*- [Bb]ackup.rdl
|
||||||
|
*- [Bb]ackup ([0-9]).rdl
|
||||||
|
*- [Bb]ackup ([0-9][0-9]).rdl
|
||||||
|
|
||||||
|
# Microsoft Fakes
|
||||||
|
FakesAssemblies/
|
||||||
|
|
||||||
|
# GhostDoc plugin setting file
|
||||||
|
*.GhostDoc.xml
|
||||||
|
|
||||||
|
# Node.js Tools for Visual Studio
|
||||||
|
.ntvs_analysis.dat
|
||||||
|
node_modules/
|
||||||
|
|
||||||
|
# Visual Studio 6 build log
|
||||||
|
*.plg
|
||||||
|
|
||||||
|
# Visual Studio 6 workspace options file
|
||||||
|
*.opt
|
||||||
|
|
||||||
|
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||||
|
*.vbw
|
||||||
|
|
||||||
|
# Visual Studio 6 auto-generated project file (contains which files were open etc.)
|
||||||
|
*.vbp
|
||||||
|
|
||||||
|
# Visual Studio 6 workspace and project file (working project files containing files to include in project)
|
||||||
|
*.dsw
|
||||||
|
*.dsp
|
||||||
|
|
||||||
|
# Visual Studio 6 technical files
|
||||||
|
*.ncb
|
||||||
|
*.aps
|
||||||
|
|
||||||
|
# Visual Studio LightSwitch build output
|
||||||
|
**/*.HTMLClient/GeneratedArtifacts
|
||||||
|
**/*.DesktopClient/GeneratedArtifacts
|
||||||
|
**/*.DesktopClient/ModelManifest.xml
|
||||||
|
**/*.Server/GeneratedArtifacts
|
||||||
|
**/*.Server/ModelManifest.xml
|
||||||
|
_Pvt_Extensions
|
||||||
|
|
||||||
|
# Paket dependency manager
|
||||||
|
.paket/paket.exe
|
||||||
|
paket-files/
|
||||||
|
|
||||||
|
# FAKE - F# Make
|
||||||
|
.fake/
|
||||||
|
|
||||||
|
# CodeRush personal settings
|
||||||
|
.cr/personal
|
||||||
|
|
||||||
|
# Python Tools for Visual Studio (PTVS)
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
||||||
|
# Cake - Uncomment if you are using it
|
||||||
|
# tools/**
|
||||||
|
# !tools/packages.config
|
||||||
|
|
||||||
|
# Tabs Studio
|
||||||
|
*.tss
|
||||||
|
|
||||||
|
# Telerik's JustMock configuration file
|
||||||
|
*.jmconfig
|
||||||
|
|
||||||
|
# BizTalk build output
|
||||||
|
*.btp.cs
|
||||||
|
*.btm.cs
|
||||||
|
*.odx.cs
|
||||||
|
*.xsd.cs
|
||||||
|
|
||||||
|
# OpenCover UI analysis results
|
||||||
|
OpenCover/
|
||||||
|
|
||||||
|
# Azure Stream Analytics local run output
|
||||||
|
ASALocalRun/
|
||||||
|
|
||||||
|
# MSBuild Binary and Structured Log
|
||||||
|
*.binlog
|
||||||
|
|
||||||
|
# NVidia Nsight GPU debugger configuration file
|
||||||
|
*.nvuser
|
||||||
|
|
||||||
|
# MFractors (Xamarin productivity tool) working folder
|
||||||
|
.mfractor/
|
||||||
|
|
||||||
|
# Local History for Visual Studio
|
||||||
|
.localhistory/
|
||||||
|
|
||||||
|
# Visual Studio History (VSHistory) files
|
||||||
|
.vshistory/
|
||||||
|
|
||||||
|
# BeatPulse healthcheck temp database
|
||||||
|
healthchecksdb
|
||||||
|
|
||||||
|
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
||||||
|
MigrationBackup/
|
||||||
|
|
||||||
|
# Ionide (cross platform F# VS Code tools) working folder
|
||||||
|
.ionide/
|
||||||
|
|
||||||
|
# Fody - auto-generated XML schema
|
||||||
|
FodyWeavers.xsd
|
||||||
|
|
||||||
|
# VS Code files for those working on multiple tools
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/settings.json
|
||||||
|
!.vscode/tasks.json
|
||||||
|
!.vscode/launch.json
|
||||||
|
!.vscode/extensions.json
|
||||||
|
*.code-workspace
|
||||||
|
|
||||||
|
# Local History for Visual Studio Code
|
||||||
|
.history/
|
||||||
|
|
||||||
|
# Windows Installer files from build outputs
|
||||||
|
*.cab
|
||||||
|
*.msi
|
||||||
|
*.msix
|
||||||
|
*.msm
|
||||||
|
*.msp
|
||||||
|
|
||||||
|
# JetBrains Rider
|
||||||
|
*.sln.iml
|
||||||
25
azala.info Scraper.sln
Normal file
25
azala.info Scraper.sln
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
|
||||||
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
|
# Visual Studio Version 16
|
||||||
|
VisualStudioVersion = 16.0.29418.71
|
||||||
|
MinimumVisualStudioVersion = 10.0.40219.1
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "azala.info Scraper", "azala.info Scraper\azala.info Scraper.csproj", "{886207E9-C764-47F1-A5E5-1556A9E28648}"
|
||||||
|
EndProject
|
||||||
|
Global
|
||||||
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
|
Debug|Any CPU = Debug|Any CPU
|
||||||
|
Release|Any CPU = Release|Any CPU
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
|
{886207E9-C764-47F1-A5E5-1556A9E28648}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{886207E9-C764-47F1-A5E5-1556A9E28648}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{886207E9-C764-47F1-A5E5-1556A9E28648}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{886207E9-C764-47F1-A5E5-1556A9E28648}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
|
HideSolutionNode = FALSE
|
||||||
|
EndGlobalSection
|
||||||
|
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||||
|
SolutionGuid = {B230E700-F905-4365-9568-773D9A713941}
|
||||||
|
EndGlobalSection
|
||||||
|
EndGlobal
|
||||||
6
azala.info Scraper/App.config
Normal file
6
azala.info Scraper/App.config
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8" ?>
|
||||||
|
<configuration>
|
||||||
|
<startup>
|
||||||
|
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.2" />
|
||||||
|
</startup>
|
||||||
|
</configuration>
|
||||||
16
azala.info Scraper/Card.cs
Normal file
16
azala.info Scraper/Card.cs
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
using Newtonsoft.Json;
|
||||||
|
|
||||||
|
namespace azala.info_Scraper {
|
||||||
|
internal class Card {
|
||||||
|
[JsonProperty(PropertyName = "id")]
|
||||||
|
public long ID { get; set; }
|
||||||
|
[JsonProperty(PropertyName = "card_text")]
|
||||||
|
public string Text { get; set; }
|
||||||
|
[JsonProperty(PropertyName = "num_blanks")]
|
||||||
|
public int Blanks { get; set; }
|
||||||
|
[JsonProperty(PropertyName = "o")]
|
||||||
|
public int Order { get; set; }
|
||||||
|
[JsonProperty(PropertyName = "deck_id")]
|
||||||
|
public int DeckID { get; internal set; }
|
||||||
|
}
|
||||||
|
}
|
||||||
174
azala.info Scraper/Program.cs
Normal file
174
azala.info Scraper/Program.cs
Normal file
@ -0,0 +1,174 @@
|
|||||||
|
using Newtonsoft.Json;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using MySql.Data;
|
||||||
|
using MySql.Data.MySqlClient;
|
||||||
|
using System.Data;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using System.Threading;
|
||||||
|
|
||||||
|
namespace azala.info_Scraper {
|
||||||
|
public class Program {
|
||||||
|
private readonly MySqlConnection connection;
|
||||||
|
|
||||||
|
public Program() {
|
||||||
|
connection = new MySqlConnection($"Server={"localhost"};Database={"cardsagainsthumanity"};Uid={"dbuser"};Pwd={"DSKJjojofsa9%=(!=i2100"};");
|
||||||
|
connection.StateChange += Connection_StateChange;
|
||||||
|
errorstream = new StreamWriter("errors.txt", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void Connection_StateChange(object sender, StateChangeEventArgs e) {
|
||||||
|
Console.WriteLine("MySQL connection is now {0}!", e.CurrentState);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task OpenConnection() {
|
||||||
|
Console.WriteLine("Connecting to MySQL Server...");
|
||||||
|
await connection.OpenAsync();
|
||||||
|
}
|
||||||
|
~Program() {
|
||||||
|
connection.Close();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async Task Main() {
|
||||||
|
var program = new Program();
|
||||||
|
var openTask = program.OpenConnection();
|
||||||
|
Console.WriteLine("Give a directory:");
|
||||||
|
var path = XConsole.ReadDirectory();
|
||||||
|
try {
|
||||||
|
await openTask;
|
||||||
|
program.Run(path);
|
||||||
|
#pragma warning disable CA1031 // Do not catch general exception types
|
||||||
|
} catch (Exception e) {
|
||||||
|
Console.WriteLine("Uncaught ERROR: {0}", e);
|
||||||
|
}
|
||||||
|
#pragma warning restore CA1031 // Do not catch general exception types
|
||||||
|
Console.WriteLine("Program end...");
|
||||||
|
Console.ReadLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static readonly string[] searchPatterns = { "*.que", "*.ans" };
|
||||||
|
private readonly StreamWriter errorstream;
|
||||||
|
private string processeddirectory;
|
||||||
|
|
||||||
|
private void Run(string path) {
|
||||||
|
processeddirectory = Path.Combine(path, "processed");
|
||||||
|
if (connection.State != ConnectionState.Open)
|
||||||
|
throw new Exception("Connection not opened!");
|
||||||
|
Console.WriteLine("Creating tables...");
|
||||||
|
try {
|
||||||
|
CreateTables();
|
||||||
|
} catch (MySqlException e) {
|
||||||
|
Console.WriteLine("ERROR: Creating tables: {0}", e.Message);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
Console.WriteLine("Tables created!");
|
||||||
|
while (true) {
|
||||||
|
foreach (var file in XDirectory.GetFiles(path, searchPatterns, SearchOption.TopDirectoryOnly)) {
|
||||||
|
try {
|
||||||
|
ProcessFile(file);
|
||||||
|
Console.WriteLine();
|
||||||
|
} catch (MySqlException e) {
|
||||||
|
Console.WriteLine("MySQL ERROR: Processing file failed: {0}", e.Message);
|
||||||
|
} catch (ArgumentException e) {
|
||||||
|
Console.WriteLine("Argument exception: {0}", e.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Console.WriteLine("Waiting 1s..");
|
||||||
|
Thread.Sleep(1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void CreateTables() {
|
||||||
|
CreateCardsTable();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void CreateCardsTable() {
|
||||||
|
var command = connection.CreateCommand();
|
||||||
|
command.CommandText = "CREATE TABLE IF NOT EXISTS cards(id INTEGER NOT NULL PRIMARY KEY,deck_id INTEGER NOT NULL,card_text VARCHAR(2048) NOT NULL,num_blanks INTEGER NOT NULL,o INTEGER NOT NULL);";
|
||||||
|
if (command.ExecuteNonQuery() == 1)
|
||||||
|
Console.WriteLine("Table created!");
|
||||||
|
else
|
||||||
|
Console.WriteLine("Table not changed!");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ProcessFile(string file) {
|
||||||
|
var filename = Path.GetFileName(file);
|
||||||
|
Console.WriteLine("Processing file '{0}'", filename);
|
||||||
|
var file_id = Path.GetFileNameWithoutExtension(filename);
|
||||||
|
|
||||||
|
if (!Directory.Exists(processeddirectory))
|
||||||
|
Directory.CreateDirectory(processeddirectory);
|
||||||
|
|
||||||
|
Console.WriteLine("Id is {0}", file_id);
|
||||||
|
|
||||||
|
if (int.TryParse(file_id, out var deck_id)) {
|
||||||
|
var contents = File.ReadAllText(file);
|
||||||
|
var cardlist = JsonConvert.DeserializeObject<List<Card>>(contents);
|
||||||
|
var insertedIds = new HashSet<long>();
|
||||||
|
|
||||||
|
foreach (var card in cardlist) {
|
||||||
|
if (insertedIds.Contains(card.ID)) {
|
||||||
|
Console.WriteLine("Card with id '{0}' already inserted! Skipping...", card.ID);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
card.DeckID = deck_id;
|
||||||
|
AddCard(card);
|
||||||
|
insertedIds.Add(card.ID);
|
||||||
|
}
|
||||||
|
Console.WriteLine("Sucessfully inserted {0}/{1} new cards!", insertedIds.Count, cardlist.Count);
|
||||||
|
File.Move(file, Path.Combine(processeddirectory, filename));
|
||||||
|
} else {
|
||||||
|
throw new ArgumentException($"File '{filename}' with non-integer ID!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void AddCard(Card card) {
|
||||||
|
using var command = connection.CreateCommand();
|
||||||
|
command.CommandText = "INSERT IGNORE INTO cards(id,deck_id,card_text,num_blanks,o) VALUES (@id,@deckid,@text,@blanks,@order);";
|
||||||
|
command.Parameters.AddWithValue("@deckid", card.DeckID);
|
||||||
|
command.Parameters.AddWithValue("@id", card.ID);
|
||||||
|
command.Parameters.AddWithValue("@text", card.Text);
|
||||||
|
command.Parameters.AddWithValue("@blanks", card.Blanks);
|
||||||
|
command.Parameters.AddWithValue("@order", card.Order);
|
||||||
|
var result = command.ExecuteNonQuery();
|
||||||
|
if (result != 1) {
|
||||||
|
Console.WriteLine("ERROR: {0} lines affected!", result);
|
||||||
|
errorstream.WriteLine("[{0}] Insertion: {1} rows affected!\n{2}", DateTime.Now.ToString(), result, JsonConvert.SerializeObject(card));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal class XConsole {
|
||||||
|
internal static string ReadDirectory() {
|
||||||
|
while (true) {
|
||||||
|
var input = Console.ReadLine();
|
||||||
|
if (Directory.Exists(input))
|
||||||
|
return input;
|
||||||
|
Console.WriteLine("Directory doesn't exist!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
public static class XDirectory { // Regex version
|
||||||
|
public static IEnumerable<string> GetFiles(string path,
|
||||||
|
string searchPatternExpression = "",
|
||||||
|
SearchOption searchOption = SearchOption.TopDirectoryOnly) {
|
||||||
|
var reSearchPattern = new Regex(searchPatternExpression, RegexOptions.IgnoreCase);
|
||||||
|
return Directory.EnumerateFiles(path, "*", searchOption)
|
||||||
|
.Where(file =>
|
||||||
|
reSearchPattern.IsMatch(Path.GetExtension(file)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Takes same patterns, and executes in parallel
|
||||||
|
public static IEnumerable<string> GetFiles(string path,
|
||||||
|
string[] searchPatterns,
|
||||||
|
SearchOption searchOption = SearchOption.TopDirectoryOnly) {
|
||||||
|
return searchPatterns.AsParallel()
|
||||||
|
.SelectMany(searchPattern =>
|
||||||
|
Directory.EnumerateFiles(path, searchPattern, searchOption));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
56
azala.info Scraper/Program.cs.old
Normal file
56
azala.info Scraper/Program.cs.old
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net.WebSockets;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Websocket.Client;
|
||||||
|
|
||||||
|
namespace azala.info_Scraper {
|
||||||
|
class Program {
|
||||||
|
private const string AZALA_INFO_URL = "wss://azala.info/main";
|
||||||
|
private readonly WebsocketClient socket;
|
||||||
|
|
||||||
|
public Program(string url) {
|
||||||
|
Console.WriteLine("Connecting to {0}", url);
|
||||||
|
var uri = new Uri(url);
|
||||||
|
|
||||||
|
socket = new WebsocketClient(uri) {
|
||||||
|
ReconnectTimeoutMs = (int)TimeSpan.FromSeconds(30).TotalMilliseconds
|
||||||
|
};
|
||||||
|
socket.ReconnectionHappened.Subscribe(type =>
|
||||||
|
Console.WriteLine($"Reconnection happened, type: {type}"));
|
||||||
|
socket.MessageReceived.Subscribe(MessageReceived);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void MessageReceived(ResponseMessage msg) {
|
||||||
|
if (msg.MessageType == WebSocketMessageType.Text) {
|
||||||
|
TextMessage(msg.Text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void TextMessage(string text) {
|
||||||
|
try {
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
Console.WriteLine("Cannot parse message (JSON expected): {0}",e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task Main() {
|
||||||
|
try {
|
||||||
|
await new Program(AZALA_INFO_URL).Run();
|
||||||
|
#pragma warning disable CA1031 // Do not catch general exception types
|
||||||
|
} catch (Exception e) {
|
||||||
|
Console.WriteLine("Uncaught error:\n{0}", e);
|
||||||
|
}
|
||||||
|
#pragma warning restore CA1031 // Do not catch general exception types
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task Run() {
|
||||||
|
await socket.Start();
|
||||||
|
Console.ReadLine();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
36
azala.info Scraper/Properties/AssemblyInfo.cs
Normal file
36
azala.info Scraper/Properties/AssemblyInfo.cs
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
using System.Reflection;
|
||||||
|
using System.Runtime.CompilerServices;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
|
// Allgemeine Informationen über eine Assembly werden über die folgenden
|
||||||
|
// Attribute gesteuert. Ändern Sie diese Attributwerte, um die Informationen zu ändern,
|
||||||
|
// die einer Assembly zugeordnet sind.
|
||||||
|
[assembly: AssemblyTitle("azala.info Scraper")]
|
||||||
|
[assembly: AssemblyDescription("")]
|
||||||
|
[assembly: AssemblyConfiguration("")]
|
||||||
|
[assembly: AssemblyCompany("")]
|
||||||
|
[assembly: AssemblyProduct("azala.info Scraper")]
|
||||||
|
[assembly: AssemblyCopyright("Copyright © 2019")]
|
||||||
|
[assembly: AssemblyTrademark("")]
|
||||||
|
[assembly: AssemblyCulture("")]
|
||||||
|
|
||||||
|
// Durch Festlegen von ComVisible auf FALSE werden die Typen in dieser Assembly
|
||||||
|
// für COM-Komponenten unsichtbar. Wenn Sie auf einen Typ in dieser Assembly von
|
||||||
|
// COM aus zugreifen müssen, sollten Sie das ComVisible-Attribut für diesen Typ auf "True" festlegen.
|
||||||
|
[assembly: ComVisible(false)]
|
||||||
|
|
||||||
|
// Die folgende GUID bestimmt die ID der Typbibliothek, wenn dieses Projekt für COM verfügbar gemacht wird
|
||||||
|
[assembly: Guid("886207e9-c764-47f1-a5e5-1556a9e28648")]
|
||||||
|
|
||||||
|
// Versionsinformationen für eine Assembly bestehen aus den folgenden vier Werten:
|
||||||
|
//
|
||||||
|
// Hauptversion
|
||||||
|
// Nebenversion
|
||||||
|
// Buildnummer
|
||||||
|
// Revision
|
||||||
|
//
|
||||||
|
// Sie können alle Werte angeben oder Standardwerte für die Build- und Revisionsnummern verwenden,
|
||||||
|
// indem Sie "*" wie unten gezeigt eingeben:
|
||||||
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
|
[assembly: AssemblyVersion("1.0.0.0")]
|
||||||
|
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||||
90
azala.info Scraper/azala.info Scraper.csproj
Normal file
90
azala.info Scraper/azala.info Scraper.csproj
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
|
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||||
|
<PropertyGroup>
|
||||||
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
|
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||||
|
<ProjectGuid>{886207E9-C764-47F1-A5E5-1556A9E28648}</ProjectGuid>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<RootNamespace>azala.info_Scraper</RootNamespace>
|
||||||
|
<AssemblyName>azala.info Scraper</AssemblyName>
|
||||||
|
<TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
|
||||||
|
<FileAlignment>512</FileAlignment>
|
||||||
|
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
||||||
|
<Deterministic>true</Deterministic>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||||
|
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||||
|
<DebugSymbols>true</DebugSymbols>
|
||||||
|
<DebugType>full</DebugType>
|
||||||
|
<Optimize>false</Optimize>
|
||||||
|
<OutputPath>bin\Debug\</OutputPath>
|
||||||
|
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||||
|
<ErrorReport>prompt</ErrorReport>
|
||||||
|
<WarningLevel>4</WarningLevel>
|
||||||
|
<LangVersion>8.0</LangVersion>
|
||||||
|
</PropertyGroup>
|
||||||
|
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||||
|
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||||
|
<DebugType>pdbonly</DebugType>
|
||||||
|
<Optimize>true</Optimize>
|
||||||
|
<OutputPath>bin\Release\</OutputPath>
|
||||||
|
<DefineConstants>TRACE</DefineConstants>
|
||||||
|
<ErrorReport>prompt</ErrorReport>
|
||||||
|
<WarningLevel>4</WarningLevel>
|
||||||
|
<LangVersion>8.0</LangVersion>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<Reference Include="BouncyCastle.Crypto, Version=1.8.3.0, Culture=neutral, PublicKeyToken=0e99375e54769942">
|
||||||
|
<HintPath>..\packages\BouncyCastle.1.8.3.1\lib\BouncyCastle.Crypto.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Google.Protobuf, Version=3.6.1.0, Culture=neutral, PublicKeyToken=a7d26565bac4d604, processorArchitecture=MSIL">
|
||||||
|
<HintPath>..\packages\Google.Protobuf.3.6.1\lib\net45\Google.Protobuf.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="MySql.Data, Version=8.0.18.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL">
|
||||||
|
<HintPath>..\packages\MySql.Data.8.0.18\lib\net452\MySql.Data.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
|
<HintPath>..\packages\Newtonsoft.Json.12.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Renci.SshNet, Version=2016.1.0.0, Culture=neutral, PublicKeyToken=1cee9f8bde3db106, processorArchitecture=MSIL">
|
||||||
|
<HintPath>..\packages\SSH.NET.2016.1.0\lib\net40\Renci.SshNet.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System" />
|
||||||
|
<Reference Include="System.ComponentModel" />
|
||||||
|
<Reference Include="System.ComponentModel.DataAnnotations" />
|
||||||
|
<Reference Include="System.Configuration" />
|
||||||
|
<Reference Include="System.Configuration.Install" />
|
||||||
|
<Reference Include="System.Core" />
|
||||||
|
<Reference Include="System.Drawing" />
|
||||||
|
<Reference Include="System.Drawing.Design" />
|
||||||
|
<Reference Include="System.Management" />
|
||||||
|
<Reference Include="System.Reactive, Version=4.0.0.0, Culture=neutral, PublicKeyToken=94bc3704cddfc263, processorArchitecture=MSIL">
|
||||||
|
<HintPath>..\packages\System.Reactive.4.0.0\lib\net46\System.Reactive.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Transactions" />
|
||||||
|
<Reference Include="System.Windows" />
|
||||||
|
<Reference Include="System.Windows.Forms" />
|
||||||
|
<Reference Include="System.Xml.Linq" />
|
||||||
|
<Reference Include="System.Data.DataSetExtensions" />
|
||||||
|
<Reference Include="Microsoft.CSharp" />
|
||||||
|
<Reference Include="System.Data" />
|
||||||
|
<Reference Include="System.Net.Http" />
|
||||||
|
<Reference Include="System.Xml" />
|
||||||
|
<Reference Include="Websocket.Client, Version=3.2.0.0, Culture=neutral, processorArchitecture=MSIL">
|
||||||
|
<HintPath>..\packages\Websocket.Client.3.2.56\lib\netstandard2.0\Websocket.Client.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="WindowsBase" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<None Include="Program.cs.old" />
|
||||||
|
<Compile Include="Card.cs" />
|
||||||
|
<Compile Include="Program.cs" />
|
||||||
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
|
</ItemGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<None Include="App.config" />
|
||||||
|
<None Include="packages.config" />
|
||||||
|
</ItemGroup>
|
||||||
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
|
</Project>
|
||||||
10
azala.info Scraper/packages.config
Normal file
10
azala.info Scraper/packages.config
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<packages>
|
||||||
|
<package id="BouncyCastle" version="1.8.3.1" targetFramework="net472" />
|
||||||
|
<package id="Google.Protobuf" version="3.6.1" targetFramework="net472" />
|
||||||
|
<package id="MySql.Data" version="8.0.18" targetFramework="net472" />
|
||||||
|
<package id="Newtonsoft.Json" version="12.0.2" targetFramework="net472" />
|
||||||
|
<package id="SSH.NET" version="2016.1.0" targetFramework="net472" />
|
||||||
|
<package id="System.Reactive" version="4.0.0" targetFramework="net472" />
|
||||||
|
<package id="Websocket.Client" version="3.2.56" targetFramework="net472" />
|
||||||
|
</packages>
|
||||||
Loading…
Reference in New Issue
Block a user