mirror of
https://github.com/bbilly1/tilefy.git
synced 2024-08-02 16:03:34 +00:00
Merge branch 'feature-cache-timeout'
This commit is contained in:
commit
e103b4ec72
20
README.md
20
README.md
@ -38,11 +38,11 @@ Main Python application to create and serve your tiles, built with Flask.
|
|||||||
- Set your timezone with the `TZ` environment variable to configure the scheduler, defaults to *UTC*.
|
- Set your timezone with the `TZ` environment variable to configure the scheduler, defaults to *UTC*.
|
||||||
|
|
||||||
### Redis JSON
|
### Redis JSON
|
||||||
Functions as a cache and holds the scheduler data storage and history.
|
Functions as a cache, holds your configurations.
|
||||||
- Needs a volume at **/data** to store your configurations permanently.
|
- Needs a volume at **/data** to store your configurations permanently.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
Create a yml config file where you have mounted your `/data/tiles.yml` folder. Take a look at the provided `tiles.example.yml` for the basic syntax. *tiles* is the top level key, list your tiles below. The main key of the tile is your slug and will become your url, so use no spaces or special characters.
|
Create a yml config file at `/data/tiles.yml`. Take a look at the provided `tiles.example.yml` for the basic syntax. *tiles* is the top level key, list your tiles below. The main key of the tile is your slug and will become your url, so use no spaces or special characters.
|
||||||
|
|
||||||
### tile_name
|
### tile_name
|
||||||
Give your tile a unique human readable name.
|
Give your tile a unique human readable name.
|
||||||
@ -76,10 +76,18 @@ Provide your custom font by adding them to `/data/fonts`, in TTF format only and
|
|||||||
Defaults to `true` for all numbers. Shorten long numbers in to a more human readable string, like *14502* to *14.5K*.
|
Defaults to `true` for all numbers. Shorten long numbers in to a more human readable string, like *14502* to *14.5K*.
|
||||||
|
|
||||||
### recreate: optional
|
### recreate: optional
|
||||||
Recreate tiles periodically, provide your custom schedule as a cron tab or use `on_demand` to recreate the tile for every request. Defaults to `0 0 * * *` aka every day at midnight. Be aware of any rate limiting and API quotas you might face with a too frequent schedule.
|
Set the lifetime of your tiles and define when the tile will be recreated if requested. Defaults to *1d*, e.g. recreate every day.
|
||||||
Note:
|
|
||||||
- There is automatically a random jitter for cron tab of 15 secs to avoid parallel requests for a lot of tiles.
|
Valid options:
|
||||||
- There is a failsafe in place to block recreating tiles faster than every 60 seconds.
|
- *120*: A number indicates seconds till expire
|
||||||
|
- *10min*: Minutes till expire
|
||||||
|
- *2h*: Hours till expire
|
||||||
|
- *1d*: Days till expire
|
||||||
|
- *on_demand*: Will recreate for every request.
|
||||||
|
|
||||||
|
Note:
|
||||||
|
- Be aware of any rate limiting and API quotas you might face with a too short expiration.
|
||||||
|
- There is a failsafe in place to block recreating tiles faster than every 60 seconds.
|
||||||
|
|
||||||
## API requests
|
## API requests
|
||||||
Get values from a public API by providing the url and key_map.
|
Get values from a public API by providing the url and key_map.
|
||||||
|
@ -3,6 +3,6 @@ beautifulsoup4==4.11.1
|
|||||||
flask==2.1.2
|
flask==2.1.2
|
||||||
Pillow==9.1.1
|
Pillow==9.1.1
|
||||||
PyYAML==6.0
|
PyYAML==6.0
|
||||||
redis==4.3.3
|
redis==4.3.4
|
||||||
requests==2.28.0
|
requests==2.28.0
|
||||||
uwsgi==2.0.20
|
uwsgi==2.0.20
|
||||||
|
47
tilefy/src/cache.py
Normal file
47
tilefy/src/cache.py
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
"""configure scheduled jobs"""
|
||||||
|
|
||||||
|
from redis.connection import ResponseError
|
||||||
|
from src.template import create_single_tile
|
||||||
|
from src.tilefy_redis import TilefyRedis
|
||||||
|
|
||||||
|
|
||||||
|
class CacheManager:
|
||||||
|
"""handle rebuild cache for tiles"""
|
||||||
|
|
||||||
|
SEC_MAP = {
|
||||||
|
"min": 60,
|
||||||
|
"h": 60 * 60,
|
||||||
|
"d": 60 * 60 * 24,
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(self, tilename):
|
||||||
|
self.tilename = tilename
|
||||||
|
self.tile_config = self.get_tile_config()
|
||||||
|
|
||||||
|
def get_tile_config(self):
|
||||||
|
"""get conf from redis"""
|
||||||
|
path = f"tiles.{self.tilename}"
|
||||||
|
try:
|
||||||
|
tile_config = TilefyRedis().get_message("config", path=path)
|
||||||
|
except ResponseError:
|
||||||
|
tile_config = False
|
||||||
|
|
||||||
|
return tile_config
|
||||||
|
|
||||||
|
def validate(self):
|
||||||
|
"""validate cache"""
|
||||||
|
key = f"lock:{self.tilename}"
|
||||||
|
use_cached = TilefyRedis().get_message(key)
|
||||||
|
if use_cached:
|
||||||
|
print(f"{self.tilename}: use cached tile")
|
||||||
|
return
|
||||||
|
|
||||||
|
create_single_tile(self.tilename, self.tile_config)
|
||||||
|
|
||||||
|
|
||||||
|
def clear_locks():
|
||||||
|
"""clear all locks from redis"""
|
||||||
|
_redis = TilefyRedis()
|
||||||
|
all_locks = _redis.get_keys("lock")
|
||||||
|
for lock in all_locks:
|
||||||
|
_redis.del_message(lock)
|
109
tilefy/src/config_parser.py
Normal file
109
tilefy/src/config_parser.py
Normal file
@ -0,0 +1,109 @@
|
|||||||
|
"""parse and load yml"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
from src.cache import clear_locks
|
||||||
|
from src.tilefy_redis import TilefyRedis
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigFile:
|
||||||
|
"""represent tile.yml file"""
|
||||||
|
|
||||||
|
TILES_CONFIG = "/data/tiles.yml"
|
||||||
|
VALID_KEYS = [
|
||||||
|
"background_color",
|
||||||
|
"font_color",
|
||||||
|
"font",
|
||||||
|
"height",
|
||||||
|
"humanize",
|
||||||
|
"key_map",
|
||||||
|
"logos",
|
||||||
|
"plugin",
|
||||||
|
"recreate",
|
||||||
|
"tile_name",
|
||||||
|
"url",
|
||||||
|
"width",
|
||||||
|
]
|
||||||
|
SEC_MAP = {
|
||||||
|
"min": 60,
|
||||||
|
"h": 60 * 60,
|
||||||
|
"d": 60 * 60 * 24,
|
||||||
|
}
|
||||||
|
MIN_EXPIRE = 60
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.exists = os.path.exists(self.TILES_CONFIG)
|
||||||
|
self.config_raw = False
|
||||||
|
self.config = False
|
||||||
|
|
||||||
|
def load_yml(self):
|
||||||
|
"""load yml into redis"""
|
||||||
|
if not self.exists:
|
||||||
|
print("missing tiles.yml")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.get_conf()
|
||||||
|
self.validate_conf()
|
||||||
|
self.add_expire()
|
||||||
|
self.save_config()
|
||||||
|
clear_locks()
|
||||||
|
|
||||||
|
def get_conf(self):
|
||||||
|
"""read config file"""
|
||||||
|
with open(self.TILES_CONFIG, "r", encoding="utf-8") as yml_file:
|
||||||
|
file_content = yml_file.read()
|
||||||
|
self.config_raw = yaml.load(file_content, Loader=yaml.CLoader)
|
||||||
|
|
||||||
|
def validate_conf(self):
|
||||||
|
"""check provided config file"""
|
||||||
|
print(f"{self.TILES_CONFIG}: validate")
|
||||||
|
all_tiles = self.config_raw.get("tiles")
|
||||||
|
if not all_tiles:
|
||||||
|
raise ValueError("missing tiles key")
|
||||||
|
|
||||||
|
for tile_name, tile_conf in all_tiles.items():
|
||||||
|
for tile_conf_key in tile_conf:
|
||||||
|
if tile_conf_key not in self.VALID_KEYS:
|
||||||
|
message = f"{tile_name}: unexpected key {tile_conf_key}"
|
||||||
|
raise ValueError(message)
|
||||||
|
|
||||||
|
self.config = self.config_raw.copy()
|
||||||
|
|
||||||
|
def add_expire(self):
|
||||||
|
"""add expire_sec to tile_conf"""
|
||||||
|
all_tiles = self.config.get("tiles")
|
||||||
|
for tile_conf in all_tiles.values():
|
||||||
|
expire = self._build_expire(tile_conf)
|
||||||
|
tile_conf.update({"recreate_sec": expire})
|
||||||
|
|
||||||
|
def _build_expire(self, tile_config):
|
||||||
|
"""validate config recreate return parsed secs"""
|
||||||
|
recreate = tile_config.get("recreate", False)
|
||||||
|
if not recreate:
|
||||||
|
return self.SEC_MAP["d"]
|
||||||
|
|
||||||
|
if isinstance(recreate, int):
|
||||||
|
if recreate < self.MIN_EXPIRE:
|
||||||
|
return self.MIN_EXPIRE
|
||||||
|
|
||||||
|
return recreate
|
||||||
|
|
||||||
|
if recreate == "on_demand":
|
||||||
|
return self.MIN_EXPIRE
|
||||||
|
|
||||||
|
try:
|
||||||
|
value, unit = re.findall(r"[a-z]+|\d+", recreate.lower())
|
||||||
|
except ValueError as err:
|
||||||
|
print(f"failed to extract value and unit of {recreate}")
|
||||||
|
raise err
|
||||||
|
|
||||||
|
if unit not in self.SEC_MAP:
|
||||||
|
raise ValueError(f"unit not in {self.SEC_MAP.keys()}")
|
||||||
|
|
||||||
|
return int(value) * self.SEC_MAP.get(unit)
|
||||||
|
|
||||||
|
def save_config(self):
|
||||||
|
"""save config in redis"""
|
||||||
|
TilefyRedis().set_message("config", self.config)
|
@ -1,109 +0,0 @@
|
|||||||
"""configure scheduled jobs"""
|
|
||||||
|
|
||||||
from os import environ
|
|
||||||
|
|
||||||
from apscheduler.jobstores.base import JobLookupError
|
|
||||||
from apscheduler.schedulers.background import BackgroundScheduler
|
|
||||||
from apscheduler.triggers.cron import CronTrigger
|
|
||||||
from src.template import create_single_tile
|
|
||||||
from src.tilefy_redis import TilefyRedis
|
|
||||||
from src.watcher import watch_yml
|
|
||||||
|
|
||||||
|
|
||||||
class TilefyScheduler:
|
|
||||||
"""interact with scheduler"""
|
|
||||||
|
|
||||||
CRON_DEFAULT = "0 0 * * *"
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.scheduler = BackgroundScheduler(timezone=environ.get("TZ", "UTC"))
|
|
||||||
self.add_job_store()
|
|
||||||
self.tiles = self.get_tiles()
|
|
||||||
|
|
||||||
def get_tiles(self):
|
|
||||||
"""get all tiles set in config"""
|
|
||||||
config = TilefyRedis().get_message("config")
|
|
||||||
if not config:
|
|
||||||
print("no tiles defined in tiles.yml")
|
|
||||||
return False
|
|
||||||
|
|
||||||
return config["tiles"]
|
|
||||||
|
|
||||||
def setup_schedule(self):
|
|
||||||
"""startup"""
|
|
||||||
if not self.tiles:
|
|
||||||
print("no tiles defined in tiles.yml")
|
|
||||||
return
|
|
||||||
|
|
||||||
jobs = self.build_jobs()
|
|
||||||
self.add_jobs(jobs)
|
|
||||||
self.add_watcher()
|
|
||||||
|
|
||||||
if not self.scheduler.running:
|
|
||||||
self.scheduler.start()
|
|
||||||
|
|
||||||
def add_job_store(self):
|
|
||||||
"""add jobstore to scheudler"""
|
|
||||||
self.scheduler.add_jobstore(
|
|
||||||
"redis",
|
|
||||||
jobs_key="tl:jobs",
|
|
||||||
run_times_key="tl:run_times",
|
|
||||||
host=environ.get("REDIS_HOST"),
|
|
||||||
port=environ.get("REDIS_PORT"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def clear_old(self):
|
|
||||||
"""remove old jobs before recreating"""
|
|
||||||
if not self.scheduler.running:
|
|
||||||
self.scheduler.start()
|
|
||||||
|
|
||||||
all_jobs = self.scheduler.get_jobs()
|
|
||||||
for job in all_jobs:
|
|
||||||
if job.id == "watcher":
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
self.scheduler.remove_job(job.id)
|
|
||||||
except JobLookupError:
|
|
||||||
print(f"failed to remove job with id {job.id}")
|
|
||||||
|
|
||||||
def build_jobs(self):
|
|
||||||
"""build list of expected jobs"""
|
|
||||||
jobs = []
|
|
||||||
for idx, (tile_slug, tile_conf) in enumerate(self.tiles.items()):
|
|
||||||
job = {
|
|
||||||
"job_id": str(idx),
|
|
||||||
"job_name": tile_slug,
|
|
||||||
"tile_conf": tile_conf,
|
|
||||||
}
|
|
||||||
jobs.append(job)
|
|
||||||
|
|
||||||
return jobs
|
|
||||||
|
|
||||||
def add_jobs(self, jobs):
|
|
||||||
"""add jobs to scheduler"""
|
|
||||||
for job in jobs:
|
|
||||||
cron_tab = job["tile_conf"].get("recreate", self.CRON_DEFAULT)
|
|
||||||
if cron_tab == "on_demand":
|
|
||||||
continue
|
|
||||||
|
|
||||||
job_name = job["job_name"]
|
|
||||||
self.scheduler.add_job(
|
|
||||||
create_single_tile,
|
|
||||||
CronTrigger.from_crontab(cron_tab),
|
|
||||||
id=job["job_id"],
|
|
||||||
name=job_name,
|
|
||||||
args=[job_name, job["tile_conf"]],
|
|
||||||
jitter=15,
|
|
||||||
replace_existing=True,
|
|
||||||
)
|
|
||||||
print(f"{job_name}: Add job {cron_tab}")
|
|
||||||
|
|
||||||
def add_watcher(self):
|
|
||||||
"""add watcher to jobs"""
|
|
||||||
self.scheduler.add_job(
|
|
||||||
watch_yml,
|
|
||||||
"interval",
|
|
||||||
seconds=5,
|
|
||||||
id="watcher",
|
|
||||||
replace_existing=True,
|
|
||||||
)
|
|
@ -1,10 +0,0 @@
|
|||||||
"""rebuild jobs in scheduler"""
|
|
||||||
|
|
||||||
from src import scheduler
|
|
||||||
|
|
||||||
|
|
||||||
def rebuild():
|
|
||||||
"""rebuild"""
|
|
||||||
handler = scheduler.TilefyScheduler()
|
|
||||||
handler.clear_old()
|
|
||||||
handler.setup_schedule()
|
|
@ -115,12 +115,19 @@ def create_all_tiles():
|
|||||||
|
|
||||||
def create_single_tile(tile_slug, tile_config):
|
def create_single_tile(tile_slug, tile_config):
|
||||||
"""create a single tile"""
|
"""create a single tile"""
|
||||||
key = f"lock:{tile_slug}"
|
|
||||||
locked = TilefyRedis().get_message(key)
|
|
||||||
if locked:
|
|
||||||
print(f"{tile_slug}: skip rebuild within 60secs")
|
|
||||||
return
|
|
||||||
|
|
||||||
TileImage(tile_slug, tile_config).build_tile()
|
TileImage(tile_slug, tile_config).build_tile()
|
||||||
message = {"recreate": int(datetime.now().strftime("%s"))}
|
|
||||||
TilefyRedis().set_message(key, message, expire=60)
|
now = datetime.now()
|
||||||
|
date_format = "%Y-%m-%d %H:%M:%S"
|
||||||
|
expire_sec = tile_config["recreate_sec"]
|
||||||
|
expire_epoch = int(now.strftime("%s")) + expire_sec
|
||||||
|
expire_str = datetime.fromtimestamp(expire_epoch).strftime(date_format)
|
||||||
|
|
||||||
|
message = {
|
||||||
|
"recreated": int(now.strftime("%s")),
|
||||||
|
"recreated_str": now.strftime(date_format),
|
||||||
|
"expire": expire_epoch,
|
||||||
|
"expire_str": expire_str,
|
||||||
|
}
|
||||||
|
|
||||||
|
TilefyRedis().set_message(f"lock:{tile_slug}", message, expire=expire_sec)
|
||||||
|
@ -4,9 +4,6 @@ import json
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
import redis
|
import redis
|
||||||
import yaml
|
|
||||||
|
|
||||||
TILES_CONFIG = "/data/tiles.yml"
|
|
||||||
|
|
||||||
|
|
||||||
class RedisBase:
|
class RedisBase:
|
||||||
@ -41,20 +38,13 @@ class TilefyRedis(RedisBase):
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def get_keys(self, key):
|
||||||
|
"""get list of all key matches"""
|
||||||
|
command = f"{self.NAME_SPACE}{key}:*"
|
||||||
|
all_keys = self.conn.execute_command("KEYS", command)
|
||||||
|
|
||||||
|
return [i.decode().split(self.NAME_SPACE)[1] for i in all_keys]
|
||||||
|
|
||||||
def del_message(self, key):
|
def del_message(self, key):
|
||||||
"""delete message from redis"""
|
"""delete message from redis"""
|
||||||
self.conn.execute_command("JSON.DEL", self.NAME_SPACE + key)
|
self.conn.execute_command("JSON.DEL", self.NAME_SPACE + key)
|
||||||
|
|
||||||
|
|
||||||
def load_yml():
|
|
||||||
"""read yml file"""
|
|
||||||
|
|
||||||
if not os.path.exists(TILES_CONFIG):
|
|
||||||
print("missing tiles.yml")
|
|
||||||
return
|
|
||||||
|
|
||||||
with open(TILES_CONFIG, "r", encoding="utf-8") as yml_file:
|
|
||||||
file_content = yml_file.read()
|
|
||||||
config_raw = yaml.load(file_content, Loader=yaml.CLoader)
|
|
||||||
|
|
||||||
TilefyRedis().set_message("config", config_raw)
|
|
||||||
|
@ -3,9 +3,9 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from src.scheduler_rebuild import rebuild
|
from src.config_parser import ConfigFile
|
||||||
from src.template import create_all_tiles
|
from src.template import create_all_tiles
|
||||||
from src.tilefy_redis import TilefyRedis, load_yml
|
from src.tilefy_redis import TilefyRedis
|
||||||
|
|
||||||
|
|
||||||
class Watcher:
|
class Watcher:
|
||||||
@ -22,10 +22,9 @@ class Watcher:
|
|||||||
modified = self.is_changed()
|
modified = self.is_changed()
|
||||||
if modified:
|
if modified:
|
||||||
print(f"{self.FILE_PATH}: modified")
|
print(f"{self.FILE_PATH}: modified")
|
||||||
load_yml()
|
ConfigFile().load_yml()
|
||||||
create_all_tiles()
|
create_all_tiles()
|
||||||
self._store_last()
|
self._store_last()
|
||||||
rebuild()
|
|
||||||
|
|
||||||
def is_changed(self):
|
def is_changed(self):
|
||||||
"""check if file has changed"""
|
"""check if file has changed"""
|
||||||
|
@ -2,17 +2,22 @@
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from flask import Flask, render_template, send_from_directory
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
from src.scheduler import TilefyScheduler
|
from flask import Flask, Response, render_template, send_from_directory
|
||||||
from src.template import create_all_tiles, create_single_tile
|
from src.cache import CacheManager
|
||||||
from src.tilefy_redis import TilefyRedis, load_yml
|
from src.config_parser import ConfigFile
|
||||||
|
from src.template import create_all_tiles
|
||||||
|
from src.tilefy_redis import TilefyRedis
|
||||||
|
from src.watcher import watch_yml
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
|
ConfigFile().load_yml()
|
||||||
load_yml()
|
|
||||||
TilefyScheduler().setup_schedule()
|
|
||||||
create_all_tiles()
|
create_all_tiles()
|
||||||
|
|
||||||
|
scheduler = BackgroundScheduler(timezone=os.environ.get("TZ", "UTC"))
|
||||||
|
scheduler.add_job(watch_yml, "interval", seconds=5)
|
||||||
|
scheduler.start()
|
||||||
|
|
||||||
|
|
||||||
@app.route("/")
|
@app.route("/")
|
||||||
def home():
|
def home():
|
||||||
@ -33,13 +38,16 @@ def home():
|
|||||||
def get_tile(tile_path):
|
def get_tile(tile_path):
|
||||||
"""return tile as image"""
|
"""return tile as image"""
|
||||||
tilename = os.path.splitext(tile_path)[0]
|
tilename = os.path.splitext(tile_path)[0]
|
||||||
tile_config = TilefyRedis().get_message("config", path=f"tiles.{tilename}")
|
|
||||||
recreate = tile_config.get("recreate")
|
cache_handler = CacheManager(tilename)
|
||||||
if recreate == "on_demand":
|
if not cache_handler.tile_config:
|
||||||
create_single_tile(tilename, tile_config)
|
print(f"tile not found: {tilename}")
|
||||||
|
return Response("tile not found", status=404)
|
||||||
|
|
||||||
|
cache_handler.validate()
|
||||||
|
|
||||||
return send_from_directory(
|
return send_from_directory(
|
||||||
directory="/data/tiles",
|
directory="/data/tiles",
|
||||||
path=tile_path,
|
path=tile_path,
|
||||||
cache_timeout=100,
|
cache_timeout=60,
|
||||||
)
|
)
|
||||||
|
@ -13,7 +13,7 @@ tiles:
|
|||||||
url: https://hub.docker.com/v2/repositories/bbilly1/tubearchivist/
|
url: https://hub.docker.com/v2/repositories/bbilly1/tubearchivist/
|
||||||
key_map:
|
key_map:
|
||||||
- pull_count
|
- pull_count
|
||||||
recreate: "0 * * * *"
|
recreate: "1d"
|
||||||
tubearchivist-github-star:
|
tubearchivist-github-star:
|
||||||
tile_name: Tube Archivist GitHub Stars
|
tile_name: Tube Archivist GitHub Stars
|
||||||
background_color: "#00202f"
|
background_color: "#00202f"
|
||||||
@ -28,7 +28,7 @@ tiles:
|
|||||||
- stargazers_count
|
- stargazers_count
|
||||||
humanize: false
|
humanize: false
|
||||||
font: ttf-bitstream-vera/VeraMono.ttf
|
font: ttf-bitstream-vera/VeraMono.ttf
|
||||||
recreate: "1 * * * *"
|
recreate: "12h"
|
||||||
tubearchivist-firefox:
|
tubearchivist-firefox:
|
||||||
tile_name: TA Companion Firefox users
|
tile_name: TA Companion Firefox users
|
||||||
background_color: "#00202f"
|
background_color: "#00202f"
|
||||||
|
Loading…
Reference in New Issue
Block a user