#!/usr/bin/env python3 import os import re import tarfile import tempfile import threading import time import subprocess from http.server import BaseHTTPRequestHandler, HTTPServer from urllib.parse import quote import requests from prometheus_client import ( CollectorRegistry, Gauge, generate_latest, ) # ============================================================ # Configuration # ============================================================ GIT_URL = "https://git.kaz.bzh/KAZ/KazV2.git" GIT_REF = "master" PORT = 9911 INVENTORY_INTERVAL = 3600 REGISTRY_CACHE_TTL = 6 * 3600 DOCKER_HUB_REGISTRY_API = "https://registry-1.docker.io" session = requests.Session() session.headers.update( { "User-Agent": "KAZ-docker-inventaire/1.0", } ) registry_cache = {} # Cache des tokens : # # (registry, repository) -> { # "token": "...", # "expires_at": timestamp # } # registry_tokens = {} print_lock = threading.Lock() # ============================================================ # Logs # ============================================================ _original_print = print def log(*args, **kwargs): timestamp = time.strftime( "%Y-%m-%d %H:%M:%S +0000", time.gmtime(), ) with print_lock: _original_print( f"[{timestamp}]", *args, **kwargs, ) print = log # ============================================================ # Prometheus # ============================================================ registry = CollectorRegistry() # ------------------------------------------------------------ # Version déclarée dans Git # ------------------------------------------------------------ git_metric = Gauge( "docker_image_git_version_info", "Docker image version declared in Git", [ "project", "service", "image", "version", ], registry=registry, ) # ------------------------------------------------------------ # Dernière version stable disponible dans le registry # ------------------------------------------------------------ source_metric = Gauge( "docker_image_source_version_info", "Latest stable Docker image version available from source registry", [ "image", "version", "registry", ], registry=registry, ) # ------------------------------------------------------------ # Métrique complète destinée à Grafana # ------------------------------------------------------------ version_metric = Gauge( "docker_image_version_info", "Docker image installed version and latest stable available version", [ "project", "service", "image", "installed_version", "available_version", "registry", ], registry=registry, ) # ------------------------------------------------------------ # Indicateur de mise à jour disponible # ------------------------------------------------------------ update_metric = Gauge( "docker_image_version_update_available", "Docker image has a newer stable version available", [ "project", "service", "image", "installed_version", "available_version", "registry", ], registry=registry, ) # ============================================================ # Version parsing # ============================================================ def parse_version(tag): """ Extrait une version numérique située au début du tag. """ if not tag: return None tag = tag.strip() match = re.match( r"^v?(\d+(?:[._-]\d+)*)(?![A-Za-z0-9])", tag, ) if not match: return None value = match.group(1) numbers = [ int(x) for x in re.split( r"[._-]", value, ) ] if numbers[0] > 999: return None return tuple(numbers) def get_tag_suffix(tag): """ Retourne ce qui se trouve après la version numérique. """ if not tag: return "" match = re.match( r"^v?(\d+(?:[._-]\d+)*)(?![A-Za-z0-9])(.*)$", tag, ) if not match: return "" suffix = match.group(2) suffix = suffix.lstrip( "._-" ) return suffix def get_build_variant(tag): """ Identifie les variantes de distribution/build. """ suffix = get_tag_suffix(tag) if not suffix: return "" known_variants = ( "alpine", "slim", "bookworm", "bullseye", "buster", "stretch", "trixie", "jammy", "noble", "focal", "debian", "ubuntu", "ubi", "distroless", ) if suffix in known_variants: return suffix match = re.match( r"^(alpine|bookworm|bullseye|buster|trixie)" r"\d+(?:\.\d+)*$", suffix, ) if match: return match.group(1) match = re.match( r"^(debian|ubuntu|alpine|bookworm|bullseye|trixie)" r"(?:[-_]\d+(?:\.\d+)*)?$", suffix, ) if match: return match.group(1) return "" # ============================================================ # Stable tag filtering # ============================================================ def is_stable_tag(tag): """ Détermine si un tag ressemble à une release stable. """ if not tag: return False lower = tag.lower() if lower in ( "latest", "production", "stable", "edge", "nightly", "snapshot", "dev", "development", "testing", "test", ): return False if re.search( r"(^|[-_.])" r"(pr|pull|preview|dev|development|nightly|" r"snapshot|master|main|branch|commit|sha)" r"([-. _]|$)", lower, ): return False if re.search( r"(^|[-_.])" r"(rc|alpha|beta|pre|preview|canary|" r"unstable|experimental)" r"(?:[-_.]|\d|$)", lower, ): return False if re.search( r"(^|[-_.])" r"(amd64|x86_64|arm64|aarch64|armv7|armv6|" r"386|ppc64le|s390x|riscv64|" r"windows|win32|linux|darwin|" r"freebsd|solaris)" r"([-. _]|$)", lower, ): return False return True # ============================================================ # Docker image # ============================================================ def split_image(image): """ Sépare registry / repository / tag. """ image = image.strip() registry_name = "docker.io" parts = image.split("/") if len(parts) > 1 and ( "." in parts[0] or ":" in parts[0] or parts[0] == "localhost" ): registry_name = parts[0] parts = parts[1:] if len(parts) == 1: repository = ( "library/" + parts[0] ) else: repository = "/".join(parts) tag = None last = repository.rsplit( "/", 1, )[-1] if ":" in last: repository, tag = repository.rsplit( ":", 1, ) if "@" in repository: repository, digest = repository.split( "@", 1, ) tag = digest return ( registry_name, repository, tag, ) # ============================================================ # Registry token cache # ============================================================ def get_cached_token( registry_name, repository, ): """ Retourne un token encore valide depuis le cache. On garde une marge de sécurité de 60 secondes avant l'expiration réelle du token. """ cache_key = ( registry_name, repository, ) cached = registry_tokens.get( cache_key ) if not cached: return None expires_at = cached.get( "expires_at", 0, ) if time.time() >= expires_at - 60: registry_tokens.pop( cache_key, None, ) print( f" Registry : token expiré pour " f"{registry_name}/{repository}" ) return None return cached.get( "token" ) def invalidate_registry_token( registry_name, repository, ): """ Supprime explicitement un token du cache. """ cache_key = ( registry_name, repository, ) if cache_key in registry_tokens: registry_tokens.pop( cache_key, None, ) print( f" Registry : token invalidé pour " f"{registry_name}/{repository}" ) def store_registry_token( registry_name, repository, token, expires_in, ): """ Stocke un token avec sa date d'expiration. expires_in est généralement fourni par le serveur d'authentification. Par sécurité, on limite également la durée maximale du cache à 24 heures. """ try: expires_in = int( expires_in ) except ( TypeError, ValueError, ): expires_in = 300 expires_in = max( 60, min( expires_in, 24 * 3600, ), ) cache_key = ( registry_name, repository, ) registry_tokens[ cache_key ] = { "token": token, "expires_at": ( time.time() + expires_in ), } # ============================================================ # Docker Hub authentication # ============================================================ def get_dockerhub_token( repository, force_refresh=False, ): """ Obtient un token Docker Hub. Le token est conservé jusqu'à son expiration. force_refresh=True permet de forcer une nouvelle authentification, notamment après un HTTP 401 du Registry. """ if not force_refresh: token = get_cached_token( "docker.io", repository, ) if token: return token else: invalidate_registry_token( "docker.io", repository, ) url = ( "https://auth.docker.io/token" "?service=registry.docker.io" f"&scope=repository:{quote(repository, safe='')}:pull" ) print( f" Registry : nouveau token Docker Hub pour " f"{repository}" ) response = session.get( url, timeout=30, ) response.raise_for_status() data = response.json() token = data.get( "token" ) if not token: raise RuntimeError( "Docker Hub : token absent de la réponse" ) expires_in = data.get( "expires_in", 300, ) store_registry_token( "docker.io", repository, token, expires_in, ) return token # ============================================================ # GHCR authentication # ============================================================ def get_ghcr_token( repository, force_refresh=False, ): """ Obtient un token GHCR. Même mécanisme de cache que Docker Hub. """ if not force_refresh: token = get_cached_token( "ghcr.io", repository, ) if token: return token else: invalidate_registry_token( "ghcr.io", repository, ) url = ( "https://ghcr.io/token" f"?scope=repository:{quote(repository, safe='')}:pull" ) print( f" Registry : nouveau token GHCR pour " f"{repository}" ) response = session.get( url, timeout=30, ) response.raise_for_status() data = response.json() token = data.get( "token" ) if not token: raise RuntimeError( "GHCR : token absent de la réponse" ) expires_in = data.get( "expires_in", 300, ) store_registry_token( "ghcr.io", repository, token, expires_in, ) return token # ============================================================ # Registry tags # ============================================================ def get_registry_tags( registry_name, repository, ): """ Récupère TOUS les tags du registry. Docker Hub : registry-1.docker.io GHCR : ghcr.io La pagination Link peut être absolue ou relative. En cas de HTTP 401 sur un registry authentifié : 1. le token est supprimé ; 2. un nouveau token est demandé ; 3. la requête est rejouée une seule fois. """ cache_key = ( registry_name, repository, ) cached = registry_cache.get( cache_key ) if cached: timestamp, tags = cached if ( time.time() - timestamp < REGISTRY_CACHE_TTL ): print( f" Registry : cache pour " f"{registry_name}/{repository}" ) return tags print( f" Registry : récupération des tags de " f"{registry_name}/{repository}" ) if registry_name == "docker.io": base_url = ( DOCKER_HUB_REGISTRY_API ) token = get_dockerhub_token( repository ) headers = { "Authorization": ( f"Bearer {token}" ), } authenticated = True elif registry_name == "ghcr.io": base_url = "https://ghcr.io" token = get_ghcr_token( repository ) headers = { "Authorization": ( f"Bearer {token}" ), } authenticated = True else: base_url = ( f"https://{registry_name}" ) headers = {} authenticated = False url = ( f"{base_url}/v2/" f"{repository}/tags/list?n=100" ) tags = [] # -------------------------------------------------------- # Une seule tentative normale + un éventuel renouvellement # du token après 401. # -------------------------------------------------------- token_refresh_attempted = False while url: response = session.get( url, headers=headers, timeout=60, ) # ---------------------------------------------------- # Token refusé. # # On ne boucle surtout pas indéfiniment : # un seul renouvellement est tenté. # ---------------------------------------------------- if ( response.status_code == 401 and authenticated and not token_refresh_attempted ): token_refresh_attempted = True print( f" Registry : HTTP 401 pour " f"{registry_name}/{repository}" ) print( f" Registry : renouvellement du token pour " f"{registry_name}/{repository}" ) if registry_name == "docker.io": token = get_dockerhub_token( repository, force_refresh=True, ) else: token = get_ghcr_token( repository, force_refresh=True, ) headers = { "Authorization": ( f"Bearer {token}" ), } # On rejoue exactement la même requête. continue response.raise_for_status() data = response.json() tags.extend( data.get( "tags", [], ) ) next_url = None link = response.headers.get( "Link" ) if link: match = re.search( r"<([^>]+)>;\s*rel=\"next\"", link, ) if match: next_url = match.group(1) if next_url.startswith("/"): next_url = ( base_url + next_url ) url = next_url tags = list( dict.fromkeys(tags) ) registry_cache[ cache_key ] = ( time.time(), tags, ) print( f" Registry : {len(tags)} tags pour " f"{registry_name}/{repository}" ) return tags # ============================================================ # Bitnami # ============================================================ def source_version_bitnami( repository, ): """ Bitnami ne fournit pas toujours une liste de tags exploitable via Registry. On cherche les releases dans les PR GitHub. """ image_name = repository.rsplit( "/", 1, )[-1] print( f" GitHub Bitnami : recherche des versions " f"de {repository}" ) url = ( "https://api.github.com/search/issues" ) query = ( f"repo:bitnami/containers " f'"[bitnami/{image_name}] Release" ' f"is:pr" ) headers = { "Accept": ( "application/vnd.github+json" ), "X-GitHub-Api-Version": "2022-11-28", "User-Agent": ( "KAZ-docker-inventaire/1.0" ), } response = session.get( url, params={ "q": query, "per_page": 100, }, headers=headers, timeout=60, ) response.raise_for_status() data = response.json() versions = [] for item in data.get( "items", [], ): title = item.get( "title", "", ) match = re.search( rf"\[bitnami/{re.escape(image_name)}\]" r"\s+Release\s+([0-9][^ ]*)", title, ) if not match: continue full_version = match.group(1) version = parse_version( full_version ) if not version: continue if not is_stable_tag( full_version ): continue numeric = ".".join( str(x) for x in version ) versions.append( ( version, numeric, ) ) if not versions: return None versions.sort( key=lambda item: item[0] ) return versions[-1][1] # ============================================================ # Source version # ============================================================ def source_version_for( image, ): """ Retourne la dernière version stable disponible. """ ( registry_name, repository, current_tag, ) = split_image(image) if not current_tag: return None if current_tag in ( "latest", "production", ): return None current_version = parse_version( current_tag ) if not current_version: return None current_variant = get_build_variant( current_tag ) # -------------------------------------------------------- # Bitnami # -------------------------------------------------------- if ( registry_name == "docker.io" and repository.startswith( "bitnami/" ) ): try: version = ( source_version_bitnami( repository ) ) if version: print( f" {repository} -> {version}" ) return version except Exception as exc: print( f" Bitnami erreur: {exc}" ) return None # -------------------------------------------------------- # Registry # -------------------------------------------------------- try: tags = get_registry_tags( registry_name, repository, ) except Exception as exc: print( f" Registry erreur: {exc}" ) return None candidates = [] for tag in tags: if not tag: continue if tag in ( "latest", "production", ): continue if not is_stable_tag( tag ): continue version = parse_version( tag ) if not version: continue variant = get_build_variant( tag ) if current_variant: if variant != current_variant: continue exact = ( 1 if tag.startswith("v") else 0 ) no_suffix = ( 1 if get_tag_suffix(tag) == "" else 0 ) candidates.append( ( version, no_suffix, exact, tag, ) ) if not candidates: print( f" Source : aucune version compatible pour " f"{image}" ) return None candidates.sort( key=lambda item: ( item[0], item[1], item[2], ) ) source_version = candidates[-1][3] print( f" Source : {image} -> " f"{source_version}" ) return source_version # ============================================================ # Git archive # ============================================================ def git_archive(): print( f"Git : récupération de {GIT_URL}" ) tempdir = tempfile.mkdtemp( prefix="docker-inventaire-" ) archive_path = os.path.join( tempdir, "dockers.tar", ) command = [ "git", "archive", "--remote=" + GIT_URL, GIT_REF, "dockers", ] with open( archive_path, "wb", ) as archive: process = subprocess.run( command, stdout=archive, stderr=subprocess.PIPE, ) if process.returncode != 0: raise RuntimeError( "git archive a échoué : " + process.stderr.decode( errors="replace" ) ) extract_dir = os.path.join( tempdir, "src", ) os.makedirs( extract_dir, exist_ok=True, ) with tarfile.open( archive_path, "r", ) as tar: for member in tar.getmembers(): if ( member.issym() or member.islnk() ): continue tar.extract( member, extract_dir, filter="data", ) print( "Git : archive extraite" ) return extract_dir # ============================================================ # Compose parser # ============================================================ def parse_compose_file( path, ): """ Parse le compose comme du texte. """ result = [] try: with open( path, "r", encoding="utf-8", errors="replace", ) as file: lines = file.readlines() except Exception: return result current_service = None service_indent = None for line in lines: if not line.strip(): continue if line.lstrip().startswith("#"): continue indent = ( len(line) - len(line.lstrip()) ) stripped = line.strip() match_service = re.match( r"^([A-Za-z0-9_.-]+):\s*$", stripped, ) if ( match_service and indent >= 2 ): current_service = ( match_service.group(1) ) service_indent = indent continue if ( current_service and service_indent is not None and indent > service_indent ): match_image = re.match( r"^image:\s*['\"]?([^'\"\s]+)", stripped, ) if match_image: image = match_image.group( 1 ) if ( image.endswith( ":latest" ) or image.endswith( ":production" ) ): continue if ":" not in image: continue ( registry_name, repository, tag, ) = split_image(image) if not tag: continue if not parse_version(tag): continue result.append( ( current_service, image, ) ) return result # ============================================================ # Inventaire # ============================================================ def inventaire(): print( "=" * 50 ) print( "Début inventaire" ) # -------------------------------------------------------- # Nettoyage des anciennes métriques # -------------------------------------------------------- git_metric._metrics.clear() source_metric._metrics.clear() version_metric._metrics.clear() update_metric._metrics.clear() # -------------------------------------------------------- # Git # -------------------------------------------------------- try: source_dir = git_archive() except Exception as exc: print( f"Git erreur : {exc}" ) return compose_files = [] for root, dirs, files in os.walk( os.path.join( source_dir, "dockers", ) ): for filename in files: if filename in ( "docker-compose.yml", "docker-compose.yaml", "docker-compose.yml.dist", "docker-compose.yaml.dist", "docker-compose.tmpl.yml.dist", "docker-compose.tmpl.yaml.dist", ): compose_files.append( os.path.join( root, filename, ) ) print( f"Git : {len(compose_files)} fichiers compose" ) # -------------------------------------------------------- # Images # -------------------------------------------------------- images = {} for compose_file in compose_files: relative = os.path.relpath( compose_file, os.path.join( source_dir, "dockers", ), ) project = relative.split( os.sep, 1, )[0] entries = parse_compose_file( compose_file ) for service, image in entries: ( registry_name, repository, tag, ) = split_image(image) if not tag: continue git_metric.labels( project=project, service=service, image=image, version=tag, ).set(1) images.setdefault( image, [], ).append( { "project": project, "service": service, "registry": registry_name, "installed_version": tag, } ) # -------------------------------------------------------- # Suppression des doublons # -------------------------------------------------------- for image in images: unique_entries = [] seen = set() for entry in images[image]: key = ( entry["project"], entry["service"], entry["registry"], entry["installed_version"], ) if key in seen: continue seen.add(key) unique_entries.append( entry ) images[image] = unique_entries print( f"Git : {len(images)} images distinctes" ) # -------------------------------------------------------- # Source # -------------------------------------------------------- print( "Recherche des versions Source..." ) for image in sorted( images ): source_version = None try: ( registry_name, repository, current_tag, ) = split_image(image) source_version = ( source_version_for( image ) ) if source_version: source_metric.labels( image=image, version=source_version, registry=registry_name, ).set(1) except Exception as exc: print( f" Erreur Source {image}: " f"{exc}" ) # ---------------------------------------------------- # Métrique complète pour Grafana # ---------------------------------------------------- available_version = ( source_version if source_version else "" ) for entry in images[image]: installed_version = entry[ "installed_version" ] version_metric.labels( project=entry["project"], service=entry["service"], image=image, installed_version=installed_version, available_version=available_version, registry=entry["registry"], ).set(1) update_metric.labels( project=entry["project"], service=entry["service"], image=image, installed_version=installed_version, available_version=available_version, registry=entry["registry"], ).set( 1 if ( available_version and installed_version != available_version ) else 0 ) print( "Fin inventaire" ) print( "=" * 50 ) # ============================================================ # HTTP Prometheus # ============================================================ class MetricsHandler( BaseHTTPRequestHandler ): def do_GET(self): if self.path != "/metrics": self.send_response( 404 ) self.end_headers() return output = generate_latest( registry ) self.send_response( 200 ) self.send_header( "Content-Type", "text/plain; version=0.0.4", ) self.send_header( "Content-Length", str(len(output)), ) self.end_headers() self.wfile.write( output ) def log_message( self, format, *args, ): return # ============================================================ # Main # ============================================================ def main(): print( "Démarrage docker-inventaire " f"sur le port {PORT}" ) server = HTTPServer( ( "0.0.0.0", PORT, ), MetricsHandler, ) def run_inventory(): while True: try: inventaire() except Exception as exc: print( f"ERREUR inventaire : {exc}" ) time.sleep( INVENTORY_INTERVAL ) thread = threading.Thread( target=run_inventory, daemon=True, ) thread.start() server.serve_forever() if __name__ == "__main__": main()