Cache full version manifests

This commit is contained in:
Michael Chen 2022-11-15 10:05:24 +01:00
parent 099436306b
commit 93efca9f95
Signed by: cnml
GPG Key ID: 5845BF3F82D5F629
2 changed files with 28 additions and 19 deletions

2
.gitignore vendored
View File

@ -1,2 +1,2 @@
versions/ versions/
*.cache cache/

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import hashlib import hashlib
from typing import Any, Literal, NotRequired, TypedDict from typing import Any, Callable, Literal, NotRequired, TypeVar, TypedDict
from pathlib import Path from pathlib import Path
import urllib.request import urllib.request
import json import json
@ -69,23 +69,25 @@ class VersionManifest(TypedDict):
versions: list[Version] versions: list[Version]
def load_version_manifest(version: Version): T = TypeVar('T')
# Download new manifest
with urllib.request.urlopen(version["url"]) as url:
data: VersionManifestFull = json.load(url)
return data
def load_manifest(reload: bool = False, max_age: timedelta = timedelta(hours=1)): def load_json(url: str) -> Any:
manifest_cache = "manifest.cache" with urllib.request.urlopen(url) as request:
# Load cached manifest return json.load(request)
cache_path = Path("cache")
def load_cached_json(url: str, cache_file: Path, loader: Callable[[str], T] = load_json, max_age: timedelta = timedelta(days=1), reload: bool = False) -> T:
if reload: if reload:
print("Forced manifest reload.") print(f"Forced cache reload for {cache_file}!")
else: else:
try: try:
with open(manifest_cache, "rb") as f: with open(cache_file, "rb") as f:
time: datetime time: datetime
data: VersionManifest data: T
(time, data) = pickle.load(f) (time, data) = pickle.load(f)
cache_age = datetime.now() - time cache_age = datetime.now() - time
if cache_age <= max_age: if cache_age <= max_age:
@ -93,19 +95,26 @@ def load_manifest(reload: bool = False, max_age: timedelta = timedelta(hours=1))
return data return data
print(f"Cache is older than {max_age} and will be reloaded.") print(f"Cache is older than {max_age} and will be reloaded.")
except FileNotFoundError: except FileNotFoundError:
print("No cached manifest found!") print(f"File {cache_file} not cached, will be reloaded!")
# Download new manifest data = loader(url)
with urllib.request.urlopen("https://launchermeta.mojang.com/mc/game/version_manifest.json") as url:
data: VersionManifest = json.load(url)
# Save manifest to cache # Save to cache
with open(manifest_cache, "wb") as f: cache_file.parent.mkdir(parents=True, exist_ok=True)
with open(cache_file, "wb") as f:
pickle.dump((datetime.now(), data), f) pickle.dump((datetime.now(), data), f)
return data return data
def load_version_manifest(version: Version, reload: bool = False) -> VersionManifestFull:
return load_cached_json(version["url"], cache_path / f"manifest_{version['id']}.cache", reload=reload)
def load_manifest(reload: bool = False) -> VersionManifest:
return load_cached_json("https://launchermeta.mojang.com/mc/game/version_manifest.json", cache_path / "manifest.cache", reload=reload)
BUF_SIZE = 65536 BUF_SIZE = 65536