#!/usr/bin/env python3 import os import re import tarfile import tempfile import threading import time import subprocess import glob import shutil from http.server import BaseHTTPRequestHandler, HTTPServer from urllib.parse import quote import requests from prometheus_client import ( CollectorRegistry, Gauge, generate_latest, ) # ============================================================ # Récupère depuis Git le contenu du répertoire dockers/ du dépôt KAZ. # Nettoie les anciens répertoires temporaires /tmp/docker-inventaire-* à chaque nouvel inventaire. # Parcourt les fichiers docker-compose.yml et leurs variantes. # Extrait les images Docker et les versions déclarées dans les fichiers Compose. # Regroupe les images par projet et service. # Interroge les registres Docker pour récupérer les tags disponibles. # Recherche la dernière version stable détectée pour chaque image. # Compare cette version avec celle déclarée dans Git. # Publie des métriques Prometheus indiquant les versions et les mises à jour disponibles. # Répète l'inventaire toutes les heures et expose les métriques sur le port 9911 pour Prometheus et Grafana. # ============================================================ # ============================================================ # Configuration # ============================================================ GIT_URL = "https://git.kaz.bzh/KAZ/KazV2.git" GIT_REF = "master" PORT = 9911 INVENTORY_INTERVAL = 3600 REGISTRY_CACHE_TTL = 6 * 3600 DOCKER_HUB_REGISTRY_API = "https://registry-1.docker.io" session = requests.Session() session.headers.update( { "User-Agent": "KAZ-docker-inventaire/1.0", } ) registry_cache = {} # Cache des tokens : # # (registry, repository) -> { # "token": "...", # "expires_at": timestamp # } # registry_tokens = {} print_lock = threading.Lock() # ============================================================ # Logs # ============================================================ _original_print = print def log(*args, **kwargs): timestamp = time.strftime( "%Y-%m-%d %H:%M:%S +0000", time.gmtime(), ) with print_lock: _original_print( f"[{timestamp}]", *args, **kwargs, ) print = log # ============================================================ # Prometheus # ============================================================ registry = CollectorRegistry() # ------------------------------------------------------------ # Version déclarée dans Git # ------------------------------------------------------------ git_metric = Gauge( "docker_image_git_version_info", "Docker image version declared in Git", [ "project", "service", "image", "version", ], registry=registry, ) # ------------------------------------------------------------ # Dernière version stable disponible dans le registry # ------------------------------------------------------------ source_metric = Gauge( "docker_image_source_version_info", "Latest stable Docker image version available from source registry", [ "image", "version", "registry", ], registry=registry, ) # ------------------------------------------------------------ # Métrique complète destinée à Grafana # ------------------------------------------------------------ version_metric = Gauge( "docker_image_version_info", "Docker image installed version and latest stable available version", [ "project", "service", "image", "installed_version", "available_version", "registry", ], registry=registry, ) # ------------------------------------------------------------ # Indicateur de mise à jour disponible # ------------------------------------------------------------ update_metric = Gauge( "docker_image_version_update_available", "Docker image has a newer stable version available", [ "project", "service", "image", "installed_version", "available_version", "registry", ], registry=registry, ) # ============================================================ # Version parsing # ============================================================ def parse_version(tag): """ Extrait une version numérique située au début du tag. """ if not tag: return None tag = tag.strip() match = re.match( r"^v?(\d+(?:[._-]\d+)*)(?![A-Za-z0-9])", tag, ) if not match: return None value = match.group(1) numbers = [ int(x) for x in re.split( r"[._-]", value, ) ] if numbers[0] > 999: return None return tuple(numbers) def get_tag_suffix(tag): """ Retourne ce qui se trouve après la version numérique. """ if not tag: return "" match = re.match( r"^v?(\d+(?:[._-]\d+)*)(?![A-Za-z0-9])(.*)$", tag, ) if not match: return "" suffix = match.group(2) suffix = suffix.lstrip( "._-" ) return suffix def get_build_variant(tag): """ Identifie les variantes de distribution/build. """ suffix = get_tag_suffix(tag) if not suffix: return "" known_variants = ( "alpine", "slim", "bookworm", "bullseye", "buster", "stretch", "trixie", "jammy", "noble", "focal", "debian", "ubuntu", "ubi", "distroless", ) if suffix in known_variants: return suffix match = re.match( r"^(alpine|bookworm|bullseye|buster|trixie)" r"\d+(?:\.\d+)*$", suffix, ) if match: return match.group(1) match = re.match( r"^(debian|ubuntu|alpine|bookworm|bullseye|trixie)" r"(?:[-_]\d+(?:\.\d+)*)?$", suffix, ) if match: return match.group(1) return "" # ============================================================ # Stable tag filtering # ============================================================ def is_stable_tag(tag): """ Détermine si un tag ressemble à une release stable. """ if not tag: return False lower = tag.lower() if lower in ( "latest", "production", "stable", "edge", "nightly", "snapshot", "dev", "development", "testing", "test", ): return False if re.search( r"(^|[-_.])" r"(pr|pull|preview|dev|development|nightly|" r"snapshot|master|main|branch|commit|sha)" r"([-. _]|$)", lower, ): return False if re.search( r"(^|[-_.])" r"(rc|alpha|beta|pre|preview|canary|" r"unstable|experimental)" r"(?:[-_.]|\d|$)", lower, ): return False if re.search( r"(^|[-_.])" r"(amd64|x86_64|arm64|aarch64|armv7|armv6|" r"386|ppc64le|s390x|riscv64|" r"windows|win32|linux|darwin|" r"freebsd|solaris)" r"([-. _]|$)", lower, ): return False return True # ============================================================ # Docker image # ============================================================ def split_image(image): """ Sépare registry / repository / tag. """ image = image.strip() registry_name = "docker.io" parts = image.split("/") if len(parts) > 1 and ( "." in parts[0] or ":" in parts[0] or parts[0] == "localhost" ): registry_name = parts[0] parts = parts[1:] if len(parts) == 1: repository = ( "library/" + parts[0] ) else: repository = "/".join(parts) tag = None last = repository.rsplit( "/", 1, )[-1] if ":" in last: repository, tag = repository.rsplit( ":", 1, ) if "@" in repository: repository, digest = repository.split( "@", 1, ) tag = digest return ( registry_name, repository, tag, ) # ============================================================ # Registry token cache # ============================================================ def get_cached_token( registry_name, repository, ): """ Retourne un token encore valide depuis le cache. On garde une marge de sécurité de 60 secondes avant l'expiration réelle du token. """ cache_key = ( registry_name, repository, ) cached = registry_tokens.get( cache_key ) if not cached: return None expires_at = cached.get( "expires_at", 0, ) if time.time() >= expires_at - 60: registry_tokens.pop( cache_key, None, ) print( f" Registry : token expiré pour " f"{registry_name}/{repository}" ) return None return cached.get( "token" ) def invalidate_registry_token( registry_name, repository, ): """ Supprime explicitement un token du cache. """ cache_key = ( registry_name, repository, ) if cache_key in registry_tokens: registry_tokens.pop( cache_key, None, ) print( f" Registry : token invalidé pour " f"{registry_name}/{repository}" ) def store_registry_token( registry_name, repository, token, expires_in, ): """ Stocke un token avec sa date d'expiration. expires_in est généralement fourni par le serveur d'authentification. Par sécurité, on limite également la durée maximale du cache à 24 heures. """ try: expires_in = int( expires_in ) except ( TypeError, ValueError, ): expires_in = 300 expires_in = max( 60, min( expires_in, 24 * 3600, ), ) cache_key = ( registry_name, repository, ) registry_tokens[ cache_key ] = { "token": token, "expires_at": ( time.time() + expires_in ), } # ============================================================ # Docker Hub authentication # ============================================================ def get_dockerhub_token( repository, force_refresh=False, ): """ Obtient un token Docker Hub. Le token est conservé jusqu'à son expiration. force_refresh=True permet de forcer une nouvelle authentification, notamment après un HTTP 401 du Registry. """ if not force_refresh: token = get_cached_token( "docker.io", repository, ) if token: return token else: invalidate_registry_token( "docker.io", repository, ) url = ( "https://auth.docker.io/token" "?service=registry.docker.io" f"&scope=repository:{quote(repository, safe='')}:pull" ) print( f" Registry : nouveau token Docker Hub pour " f"{repository}" ) response = session.get( url, timeout=30, ) response.raise_for_status() data = response.json() token = data.get( "token" ) if not token: raise RuntimeError( "Docker Hub : token absent de la réponse" ) expires_in = data.get( "expires_in", 300, ) store_registry_token( "docker.io", repository, token, expires_in, ) return token # ============================================================ # GHCR authentication # ============================================================ def get_ghcr_token( repository, force_refresh=False, ): """ Obtient un token GHCR. Même mécanisme de cache que Docker Hub. """ if not force_refresh: token = get_cached_token( "ghcr.io", repository, ) if token: return token else: invalidate_registry_token( "ghcr.io", repository, ) url = ( "https://ghcr.io/token" f"?scope=repository:{quote(repository, safe='')}:pull" ) print( f" Registry : nouveau token GHCR pour " f"{repository}" ) response = session.get( url, timeout=30, ) response.raise_for_status() data = response.json() token = data.get( "token" ) if not token: raise RuntimeError( "GHCR : token absent de la réponse" ) expires_in = data.get( "expires_in", 300, ) store_registry_token( "ghcr.io", repository, token, expires_in, ) return token # ============================================================ # Registry tags # ============================================================ def get_registry_tags( registry_name, repository, ): """ Récupère TOUS les tags du registry. Docker Hub : registry-1.docker.io GHCR : ghcr.io La pagination Link peut être absolue ou relative. En cas de HTTP 401 sur un registry authentifié : 1. le token est supprimé ; 2. un nouveau token est demandé ; 3. la requête est rejouée une seule fois. """ cache_key = ( registry_name, repository, ) cached = registry_cache.get( cache_key ) if cached: timestamp, tags = cached if ( time.time() - timestamp < REGISTRY_CACHE_TTL ): print( f" Registry : cache pour " f"{registry_name}/{repository}" ) return tags print( f" Registry : récupération des tags de " f"{registry_name}/{repository}" ) if registry_name == "docker.io": base_url = ( DOCKER_HUB_REGISTRY_API ) token = get_dockerhub_token( repository ) headers = { "Authorization": ( f"Bearer {token}" ), } authenticated = True elif registry_name == "ghcr.io": base_url = "https://ghcr.io" token = get_ghcr_token( repository ) headers = { "Authorization": ( f"Bearer {token}" ), } authenticated = True else: base_url = ( f"https://{registry_name}" ) headers = {} authenticated = False url = ( f"{base_url}/v2/" f"{repository}/tags/list?n=100" ) tags = [] # -------------------------------------------------------- # Une seule tentative normale + un éventuel renouvellement # du token après 401. # -------------------------------------------------------- token_refresh_attempted = False while url: response = session.get( url, headers=headers, timeout=60, ) # ---------------------------------------------------- # Token refusé. # # On ne boucle surtout pas indéfiniment : # un seul renouvellement est tenté. # ---------------------------------------------------- if ( response.status_code == 401 and authenticated and not token_refresh_attempted ): token_refresh_attempted = True print( f" Registry : HTTP 401 pour " f"{registry_name}/{repository}" ) print( f" Registry : renouvellement du token pour " f"{registry_name}/{repository}" ) if registry_name == "docker.io": token = get_dockerhub_token( repository, force_refresh=True, ) else: token = get_ghcr_token( repository, force_refresh=True, ) headers = { "Authorization": ( f"Bearer {token}" ), } # On rejoue exactement la même requête. continue response.raise_for_status() data = response.json() tags.extend( data.get( "tags", [], ) ) next_url = None link = response.headers.get( "Link" ) if link: match = re.search( r"<([^>]+)>;\s*rel=\"next\"", link, ) if match: next_url = match.group(1) if next_url.startswith("/"): next_url = ( base_url + next_url ) url = next_url tags = list( dict.fromkeys(tags) ) registry_cache[ cache_key ] = ( time.time(), tags, ) print( f" Registry : {len(tags)} tags pour " f"{registry_name}/{repository}" ) return tags # ============================================================ # Bitnami # ============================================================ def source_version_bitnami( repository, ): """ Bitnami ne fournit pas toujours une liste de tags exploitable via Registry. On cherche les releases dans les PR GitHub. """ image_name = repository.rsplit( "/", 1, )[-1] print( f" GitHub Bitnami : recherche des versions " f"de {repository}" ) url = ( "https://api.github.com/search/issues" ) query = ( f"repo:bitnami/containers " f'"[bitnami/{image_name}] Release" ' f"is:pr" ) headers = { "Accept": ( "application/vnd.github+json" ), "X-GitHub-Api-Version": "2022-11-28", "User-Agent": ( "KAZ-docker-inventaire/1.0" ), } response = session.get( url, params={ "q": query, "per_page": 100, }, headers=headers, timeout=60, ) response.raise_for_status() data = response.json() versions = [] for item in data.get( "items", [], ): title = item.get( "title", "", ) match = re.search( rf"\[bitnami/{re.escape(image_name)}\]" r"\s+Release\s+([0-9][^ ]*)", title, ) if not match: continue full_version = match.group(1) version = parse_version( full_version ) if not version: continue if not is_stable_tag( full_version ): continue numeric = ".".join( str(x) for x in version ) versions.append( ( version, numeric, ) ) if not versions: return None versions.sort( key=lambda item: item[0] ) return versions[-1][1] # ============================================================ # Source version # ============================================================ def source_version_for( image, ): """ Retourne la dernière version stable disponible. """ ( registry_name, repository, current_tag, ) = split_image(image) if not current_tag: return None if current_tag in ( "latest", "production", ): return None current_version = parse_version( current_tag ) if not current_version: return None current_variant = get_build_variant( current_tag ) # -------------------------------------------------------- # Bitnami # -------------------------------------------------------- if ( registry_name == "docker.io" and repository.startswith( "bitnami/" ) ): try: version = ( source_version_bitnami( repository ) ) if version: print( f" {repository} -> {version}" ) return version except Exception as exc: print( f" Bitnami erreur: {exc}" ) return None # -------------------------------------------------------- # Registry # -------------------------------------------------------- try: tags = get_registry_tags( registry_name, repository, ) except Exception as exc: print( f" Registry erreur: {exc}" ) return None candidates = [] for tag in tags: if not tag: continue if tag in ( "latest", "production", ): continue if not is_stable_tag( tag ): continue version = parse_version( tag ) if not version: continue variant = get_build_variant( tag ) if current_variant: if variant != current_variant: continue exact = ( 1 if tag.startswith("v") else 0 ) no_suffix = ( 1 if get_tag_suffix(tag) == "" else 0 ) candidates.append( ( version, no_suffix, exact, tag, ) ) if not candidates: print( f" Source : aucune version compatible pour " f"{image}" ) return None candidates.sort( key=lambda item: ( item[0], item[1], item[2], ) ) source_version = candidates[-1][3] print( f" Source : {image} -> " f"{source_version}" ) return source_version # ============================================================ # Git archive # ============================================================ def git_archive(): print( f"Git : récupération de {GIT_URL}" ) # Nettoyage des archives temporaires précédentes for path in glob.glob("/tmp/docker-inventaire-*"): if os.path.isdir(path): shutil.rmtree(path, ignore_errors=True) tempdir = tempfile.mkdtemp( prefix="docker-inventaire-" ) archive_path = os.path.join( tempdir, "dockers.tar", ) command = [ "git", "archive", "--remote=" + GIT_URL, GIT_REF, "dockers", ] with open( archive_path, "wb", ) as archive: process = subprocess.run( command, stdout=archive, stderr=subprocess.PIPE, ) if process.returncode != 0: raise RuntimeError( "git archive a échoué : " + process.stderr.decode( errors="replace" ) ) extract_dir = os.path.join( tempdir, "src", ) os.makedirs( extract_dir, exist_ok=True, ) with tarfile.open( archive_path, "r", ) as tar: for member in tar.getmembers(): if ( member.issym() or member.islnk() ): continue tar.extract( member, extract_dir, filter="data", ) print( "Git : archive extraite" ) return extract_dir # ============================================================ # Compose parser # ============================================================ def parse_compose_file( path, ): """ Parse le compose comme du texte. """ result = [] try: with open( path, "r", encoding="utf-8", errors="replace", ) as file: lines = file.readlines() except Exception: return result current_service = None service_indent = None for line in lines: if not line.strip(): continue if line.lstrip().startswith("#"): continue indent = ( len(line) - len(line.lstrip()) ) stripped = line.strip() match_service = re.match( r"^([A-Za-z0-9_.-]+):\s*$", stripped, ) if ( match_service and indent >= 2 ): current_service = ( match_service.group(1) ) service_indent = indent continue if ( current_service and service_indent is not None and indent > service_indent ): match_image = re.match( r"^image:\s*['\"]?([^'\"\s]+)", stripped, ) if match_image: image = match_image.group( 1 ) if ( image.endswith( ":latest" ) or image.endswith( ":production" ) ): continue if ":" not in image: continue ( registry_name, repository, tag, ) = split_image(image) if not tag: continue if not parse_version(tag): continue result.append( ( current_service, image, ) ) return result # ============================================================ # Inventaire # ============================================================ def inventaire(): print( "=" * 50 ) print( "Début inventaire" ) # -------------------------------------------------------- # Nettoyage des anciennes métriques # -------------------------------------------------------- git_metric._metrics.clear() source_metric._metrics.clear() version_metric._metrics.clear() update_metric._metrics.clear() # -------------------------------------------------------- # Git # -------------------------------------------------------- try: source_dir = git_archive() except Exception as exc: print( f"Git erreur : {exc}" ) return compose_files = [] for root, dirs, files in os.walk( os.path.join( source_dir, "dockers", ) ): for filename in files: if filename in ( "docker-compose.yml", "docker-compose.yaml", "docker-compose.yml.dist", "docker-compose.yaml.dist", "docker-compose.tmpl.yml.dist", "docker-compose.tmpl.yaml.dist", ): compose_files.append( os.path.join( root, filename, ) ) print( f"Git : {len(compose_files)} fichiers compose" ) # -------------------------------------------------------- # Images # -------------------------------------------------------- images = {} for compose_file in compose_files: relative = os.path.relpath( compose_file, os.path.join( source_dir, "dockers", ), ) project = relative.split( os.sep, 1, )[0] entries = parse_compose_file( compose_file ) for service, image in entries: ( registry_name, repository, tag, ) = split_image(image) if not tag: continue git_metric.labels( project=project, service=service, image=image, version=tag, ).set(1) images.setdefault( image, [], ).append( { "project": project, "service": service, "registry": registry_name, "installed_version": tag, } ) # -------------------------------------------------------- # Suppression des doublons # -------------------------------------------------------- for image in images: unique_entries = [] seen = set() for entry in images[image]: key = ( entry["project"], entry["service"], entry["registry"], entry["installed_version"], ) if key in seen: continue seen.add(key) unique_entries.append( entry ) images[image] = unique_entries print( f"Git : {len(images)} images distinctes" ) # -------------------------------------------------------- # Source # -------------------------------------------------------- print( "Recherche des versions Source..." ) for image in sorted( images ): source_version = None try: ( registry_name, repository, current_tag, ) = split_image(image) source_version = ( source_version_for( image ) ) if source_version: source_metric.labels( image=image, version=source_version, registry=registry_name, ).set(1) except Exception as exc: print( f" Erreur Source {image}: " f"{exc}" ) # ---------------------------------------------------- # Métrique complète pour Grafana # ---------------------------------------------------- available_version = ( source_version if source_version else "" ) for entry in images[image]: installed_version = entry[ "installed_version" ] version_metric.labels( project=entry["project"], service=entry["service"], image=image, installed_version=installed_version, available_version=available_version, registry=entry["registry"], ).set(1) update_metric.labels( project=entry["project"], service=entry["service"], image=image, installed_version=installed_version, available_version=available_version, registry=entry["registry"], ).set( 1 if ( available_version and installed_version != available_version ) else 0 ) print( "Fin inventaire" ) print( "=" * 50 ) # ============================================================ # HTTP Prometheus # ============================================================ class MetricsHandler( BaseHTTPRequestHandler ): def do_GET(self): if self.path != "/metrics": self.send_response( 404 ) self.end_headers() return output = generate_latest( registry ) self.send_response( 200 ) self.send_header( "Content-Type", "text/plain; version=0.0.4", ) self.send_header( "Content-Length", str(len(output)), ) self.end_headers() self.wfile.write( output ) def log_message( self, format, *args, ): return # ============================================================ # Main # ============================================================ def main(): print( "Démarrage docker-inventaire " f"sur le port {PORT}" ) server = HTTPServer( ( "0.0.0.0", PORT, ), MetricsHandler, ) def run_inventory(): while True: try: inventaire() except Exception as exc: print( f"ERREUR inventaire : {exc}" ) time.sleep( INVENTORY_INTERVAL ) thread = threading.Thread( target=run_inventory, daemon=True, ) thread.start() server.serve_forever() if __name__ == "__main__": main()