Compare commits

..

3 Commits

Author SHA1 Message Date
556035bdd9
Build latest tag on main branch
clean old stuff from createreadmes
2023-03-22 17:37:00 +01:00
44bae3fa85
Added branch override for artifacts 2022-12-13 17:51:20 +01:00
62c96fcbf6
Added rule icons 2022-12-13 17:51:01 +01:00
3 changed files with 49 additions and 24 deletions

View File

@ -30,23 +30,7 @@ def get_webhook():
return None return None
if __name__ == '__main__': def update_portainer_stack(webhook_id: str):
output_path = Path("dist")
if output_path.exists():
rmtree(output_path)
output_path.mkdir()
dockerfile = Path("Dockerfile.local").resolve()
run(["python", "createreadmes.py"], check=True)
run(["bundle.bat", "exec", "jekyll", "build", "--destination", output_path], check=True)
branch = run(["git", "branch", "--show-current"],
stdout=PIPE, check=True).stdout.decode().strip()
short_sha = run(["git", "rev-parse", "--short", "HEAD"],
stdout=PIPE, check=True).stdout.decode().strip()
platforms = ['linux/amd64', 'linux/arm/v6', 'linux/arm/v7',
'linux/arm64/v8', 'linux/386', 'linux/ppc64le', 'linux/s390x']
buildx("chenio/code2dfd", [branch, short_sha], platforms, dockerfile=dockerfile)
webhook_id = get_webhook()
if webhook_id is not None:
print("Updating portainer stack...") print("Updating portainer stack...")
resp = post(f"https://docker.cnml.de/api/stacks/webhooks/{webhook_id}") resp = post(f"https://docker.cnml.de/api/stacks/webhooks/{webhook_id}")
if not resp.ok: if not resp.ok:
@ -61,3 +45,27 @@ if __name__ == '__main__':
print("Failed to update:", e) print("Failed to update:", e)
else: else:
print("Stack successfully updated!") print("Stack successfully updated!")
if __name__ == '__main__':
output_path = Path("dist")
if output_path.exists():
rmtree(output_path)
output_path.mkdir()
dockerfile = Path("Dockerfile.local").resolve()
run(["python", "createreadmes.py"], check=True)
run(["bundle.bat", "exec", "jekyll", "build", "--destination", output_path], check=True)
branch = run(["git", "branch", "--show-current"],
stdout=PIPE, check=True).stdout.decode().strip()
short_sha = run(["git", "rev-parse", "--short", "HEAD"],
stdout=PIPE, check=True).stdout.decode().strip()
tags = [branch, short_sha]
if branch == 'main':
default_branch_tag = "latest"
print(f"On default branch, also building {default_branch_tag} tag!")
tags.append(default_branch_tag)
platforms = ['linux/amd64', 'linux/arm/v6', 'linux/arm/v7',
'linux/arm64/v8', 'linux/386', 'linux/ppc64le', 'linux/s390x']
buildx("chenio/code2dfd", tags, platforms, dockerfile=dockerfile)
webhook_id = get_webhook()
if webhook_id is not None:
update_portainer_stack(webhook_id)

View File

@ -4,7 +4,7 @@ import json
import itertools import itertools
import yaml import yaml
import jsonschema import jsonschema
from typing import Any, List, NotRequired, Optional, TypedDict from typing import Any, Dict, List, Literal, NotRequired, Optional, TypedDict
import requests import requests
try: try:
from yachalk import chalk from yachalk import chalk
@ -219,9 +219,12 @@ class Artifact(TypedDict):
file: str file: str
lines: NotRequired[list[int]] lines: NotRequired[list[int]]
repository: NotRequired[str] repository: NotRequired[str]
branch: NotRequired[str]
RuleStatus = Literal["disregarded", "observed", "not applicable", "unknown"]
class SecurityRule(TypedDict): class SecurityRule(TypedDict):
status: str status: RuleStatus
argument: str | list[str] argument: str | list[str]
artifacts: NotRequired[list[Artifact]] artifacts: NotRequired[list[Artifact]]
@ -256,12 +259,14 @@ properties:
type: string type: string
repository: repository:
type: string type: string
branch:
type: string
lines: lines:
type: array type: array
items: items:
type: integer""") type: integer""")
def check_security_rules(security_rules: dict[Any, Any] | None) -> dict[int, SecurityRule]: def check_security_rules(model_id: str, security_rules: dict[Any, Any] | None) -> dict[int, SecurityRule]:
if security_rules is None: if security_rules is None:
raise Exception("Security rules file is empty!") raise Exception("Security rules file is empty!")
for n in range(1, 19): for n in range(1, 19):
@ -271,7 +276,7 @@ def check_security_rules(security_rules: dict[Any, Any] | None) -> dict[int, Sec
jsonschema.validate(rule, rule_schema) jsonschema.validate(rule, rule_schema)
rule: SecurityRule rule: SecurityRule
if rule["status"] == "unknown": if rule["status"] == "unknown":
warning(f"Rule {n} is still unknown!") warning(f"In model {model_id}: Rule {n} is still unknown!")
except jsonschema.ValidationError as e: except jsonschema.ValidationError as e:
warning("Not checking further rules!") warning("Not checking further rules!")
raise Exception("Security rule {n}: {msg} at $.{n}.{path}".format(n=n, msg=e.message, path=e.json_path)) from e raise Exception("Security rule {n}: {msg} at $.{n}.{path}".format(n=n, msg=e.message, path=e.json_path)) from e
@ -309,7 +314,9 @@ rule_names = {
def artifact_to_string(info: ModelInformation, artifact: Artifact): def artifact_to_string(info: ModelInformation, artifact: Artifact):
file = Path(artifact['file']) file = Path(artifact['file'])
filename = file.name filename = file.name
file_url = f"https://github.com/{artifact.get('repository', info['slug'])}/blob/{info.get('branch', 'master')}/{artifact['file']}" project_branch = info.get("branch", "master")
branch = artifact.get("branch", project_branch)
file_url = f"https://github.com/{artifact.get('repository', info['slug'])}/blob/{branch}/{artifact['file']}"
lines = artifact.get("lines") lines = artifact.get("lines")
if lines is None: if lines is None:
return f"- {filename}: [File]({file_url})" return f"- {filename}: [File]({file_url})"
@ -322,7 +329,7 @@ def rule_to_string(info: ModelInformation, id: int, rule: SecurityRule | None):
return "" return ""
argument = rule['argument'] argument = rule['argument']
argument = argument if isinstance(argument, str) else "".join(f"\n1. {arg}" for arg in argument) argument = argument if isinstance(argument, str) else "".join(f"\n1. {arg}" for arg in argument)
text = f"""#### Rule {id}: {rule_names[id]} text = f"""#### Rule {id}: {rule_names[id]} {{#rule{id:02}}}
This rule is {rule['status']}: {argument}""" This rule is {rule['status']}: {argument}"""
artifacts = rule.get("artifacts", []) artifacts = rule.get("artifacts", [])
@ -334,8 +341,18 @@ Artifacts:
return text return text
def write_security_rules(info: ModelInformation, security_rules: dict[int, SecurityRule]): def write_security_rules(info: ModelInformation, security_rules: dict[int, SecurityRule]):
icons: Dict[RuleStatus | str, str] = {
'disregarded': '<i class="fa fa-exclamation-circle" style="color: #d72b28;"></i>',
'observed': '<i class="fa fa-check-square-o" style="color: #6be16d;"></i>',
'not applicable': '<i class="fa fa-info-circle" style="color: #31708;"></i>',
'unknown': '<i class="fa fa-warning" style="color: #bfc600;"></i>',
}
return f"""## Security Rules return f"""## Security Rules
{" | ".join(f"R{i}" for i in range(1, 19))}
{" | ".join("--" for _ in range(1, 19))}
{" | ".join(f'<a href="#rule{i:02}">{icons[security_rules.get(i, {"status": "unknown"})["status"]]}</a>' for i in range(1, 19))}
### Authentication / Authorization ### Authentication / Authorization
{(chr(10)*2).join(rule_to_string(info, i, security_rules.get(i)) for i in range(1, 7))} {(chr(10)*2).join(rule_to_string(info, i, security_rules.get(i)) for i in range(1, 7))}
@ -404,7 +421,7 @@ def write_model_readmes(dataset: Dataset):
security_rules = None security_rules = None
try: try:
with security_rules_file.open('r') as f: with security_rules_file.open('r') as f:
security_rules = check_security_rules(yaml.safe_load(f)) security_rules = check_security_rules(model_id, yaml.safe_load(f))
except FileNotFoundError: except FileNotFoundError:
warning("Security rules file not found at {}".format(security_rules_file)) warning("Security rules file not found at {}".format(security_rules_file))
except Exception as e: except Exception as e:
@ -412,7 +429,7 @@ def write_model_readmes(dataset: Dataset):
dir.mkdir(exist_ok=True) dir.mkdir(exist_ok=True)
write_file_if_changed(readme, f"""--- write_file_if_changed(readme, f"""---
title: {slug} title: {slug}
keywords: model TODO keywords: model
tags: [{', '.join(get_tag_slug(tech) for tech in info['tech'])}] tags: [{', '.join(get_tag_slug(tech) for tech in info['tech'])}]
sidebar: datasetdoc_sidebar sidebar: datasetdoc_sidebar
permalink: {model_id}.html permalink: {model_id}.html
@ -475,7 +492,6 @@ keywords: code2DFD introduction
tags: [overview] tags: [overview]
sidebar: datasetdoc_sidebar sidebar: datasetdoc_sidebar
permalink: index.html permalink: index.html
summary: Dataset of dataflow diagrams of microservice applications.
toc: false toc: false
--- ---

View File

@ -28,6 +28,7 @@
"additionalProperties": false, "additionalProperties": false,
"properties": { "properties": {
"repository": { "type": "string" }, "repository": { "type": "string" },
"branch": { "type": "string" },
"file": { "type": "string" }, "file": { "type": "string" },
"lines": { "type": "array", "items": { "type": "integer" } } "lines": { "type": "array", "items": { "type": "integer" } }
}, },