Cache full version manifests

This commit is contained in:
Michael Chen 2022-11-15 10:05:24 +01:00
parent 099436306b
commit 93efca9f95
Signed by: cnml
GPG Key ID: 5845BF3F82D5F629
2 changed files with 28 additions and 19 deletions

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
versions/
*.cache
cache/

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
import hashlib
from typing import Any, Literal, NotRequired, TypedDict
from typing import Any, Callable, Literal, NotRequired, TypeVar, TypedDict
from pathlib import Path
import urllib.request
import json
@ -69,23 +69,25 @@ class VersionManifest(TypedDict):
versions: list[Version]
def load_version_manifest(version: Version):
# Download new manifest
with urllib.request.urlopen(version["url"]) as url:
data: VersionManifestFull = json.load(url)
return data
T = TypeVar('T')
def load_manifest(reload: bool = False, max_age: timedelta = timedelta(hours=1)):
manifest_cache = "manifest.cache"
# Load cached manifest
def load_json(url: str) -> Any:
with urllib.request.urlopen(url) as request:
return json.load(request)
cache_path = Path("cache")
def load_cached_json(url: str, cache_file: Path, loader: Callable[[str], T] = load_json, max_age: timedelta = timedelta(days=1), reload: bool = False) -> T:
if reload:
print("Forced manifest reload.")
print(f"Forced cache reload for {cache_file}!")
else:
try:
with open(manifest_cache, "rb") as f:
with open(cache_file, "rb") as f:
time: datetime
data: VersionManifest
data: T
(time, data) = pickle.load(f)
cache_age = datetime.now() - time
if cache_age <= max_age:
@ -93,19 +95,26 @@ def load_manifest(reload: bool = False, max_age: timedelta = timedelta(hours=1))
return data
print(f"Cache is older than {max_age} and will be reloaded.")
except FileNotFoundError:
print("No cached manifest found!")
print(f"File {cache_file} not cached, will be reloaded!")
# Download new manifest
with urllib.request.urlopen("https://launchermeta.mojang.com/mc/game/version_manifest.json") as url:
data: VersionManifest = json.load(url)
data = loader(url)
# Save manifest to cache
with open(manifest_cache, "wb") as f:
# Save to cache
cache_file.parent.mkdir(parents=True, exist_ok=True)
with open(cache_file, "wb") as f:
pickle.dump((datetime.now(), data), f)
return data
def load_version_manifest(version: Version, reload: bool = False) -> VersionManifestFull:
return load_cached_json(version["url"], cache_path / f"manifest_{version['id']}.cache", reload=reload)
def load_manifest(reload: bool = False) -> VersionManifest:
return load_cached_json("https://launchermeta.mojang.com/mc/game/version_manifest.json", cache_path / "manifest.cache", reload=reload)
BUF_SIZE = 65536