1753 lines
33 KiB
Python
1753 lines
33 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import re
|
|
import tarfile
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import subprocess
|
|
import glob
|
|
import shutil
|
|
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
from urllib.parse import quote
|
|
|
|
import requests
|
|
|
|
from prometheus_client import (
|
|
CollectorRegistry,
|
|
Gauge,
|
|
generate_latest,
|
|
)
|
|
|
|
# ============================================================
|
|
# Récupère depuis Git le contenu du répertoire dockers/ du dépôt KAZ.
|
|
# Nettoie les anciens répertoires temporaires /tmp/docker-inventaire-* à chaque nouvel inventaire.
|
|
# Parcourt les fichiers docker-compose.yml et leurs variantes.
|
|
# Extrait les images Docker et les versions déclarées dans les fichiers Compose.
|
|
# Regroupe les images par projet et service.
|
|
# Interroge les registres Docker pour récupérer les tags disponibles.
|
|
# Recherche la dernière version stable détectée pour chaque image.
|
|
# Compare cette version avec celle déclarée dans Git.
|
|
# Publie des métriques Prometheus indiquant les versions et les mises à jour disponibles.
|
|
# Répète l'inventaire toutes les heures et expose les métriques sur le port 9911 pour Prometheus et Grafana.
|
|
# ============================================================
|
|
|
|
|
|
# ============================================================
|
|
# Configuration
|
|
# ============================================================
|
|
|
|
GIT_URL = "https://git.kaz.bzh/KAZ/KazV2.git"
|
|
GIT_REF = "master"
|
|
|
|
PORT = 9911
|
|
|
|
INVENTORY_INTERVAL = 3600
|
|
REGISTRY_CACHE_TTL = 6 * 3600
|
|
|
|
DOCKER_HUB_REGISTRY_API = "https://registry-1.docker.io"
|
|
|
|
session = requests.Session()
|
|
|
|
session.headers.update(
|
|
{
|
|
"User-Agent": "KAZ-docker-inventaire/1.0",
|
|
}
|
|
)
|
|
|
|
registry_cache = {}
|
|
|
|
# Cache des tokens :
|
|
#
|
|
# (registry, repository) -> {
|
|
# "token": "...",
|
|
# "expires_at": timestamp
|
|
# }
|
|
#
|
|
registry_tokens = {}
|
|
|
|
print_lock = threading.Lock()
|
|
|
|
|
|
# ============================================================
|
|
# Logs
|
|
# ============================================================
|
|
|
|
_original_print = print
|
|
|
|
|
|
def log(*args, **kwargs):
|
|
timestamp = time.strftime(
|
|
"%Y-%m-%d %H:%M:%S +0000",
|
|
time.gmtime(),
|
|
)
|
|
|
|
with print_lock:
|
|
_original_print(
|
|
f"[{timestamp}]",
|
|
*args,
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
print = log
|
|
|
|
|
|
# ============================================================
|
|
# Prometheus
|
|
# ============================================================
|
|
|
|
registry = CollectorRegistry()
|
|
|
|
|
|
# ------------------------------------------------------------
|
|
# Version déclarée dans Git
|
|
# ------------------------------------------------------------
|
|
|
|
git_metric = Gauge(
|
|
"docker_image_git_version_info",
|
|
"Docker image version declared in Git",
|
|
[
|
|
"project",
|
|
"service",
|
|
"image",
|
|
"version",
|
|
],
|
|
registry=registry,
|
|
)
|
|
|
|
|
|
# ------------------------------------------------------------
|
|
# Dernière version stable disponible dans le registry
|
|
# ------------------------------------------------------------
|
|
|
|
source_metric = Gauge(
|
|
"docker_image_source_version_info",
|
|
"Latest stable Docker image version available from source registry",
|
|
[
|
|
"image",
|
|
"version",
|
|
"registry",
|
|
],
|
|
registry=registry,
|
|
)
|
|
|
|
|
|
# ------------------------------------------------------------
|
|
# Métrique complète destinée à Grafana
|
|
# ------------------------------------------------------------
|
|
|
|
version_metric = Gauge(
|
|
"docker_image_version_info",
|
|
"Docker image installed version and latest stable available version",
|
|
[
|
|
"project",
|
|
"service",
|
|
"image",
|
|
"installed_version",
|
|
"available_version",
|
|
"registry",
|
|
],
|
|
registry=registry,
|
|
)
|
|
|
|
|
|
# ------------------------------------------------------------
|
|
# Indicateur de mise à jour disponible
|
|
# ------------------------------------------------------------
|
|
|
|
update_metric = Gauge(
|
|
"docker_image_version_update_available",
|
|
"Docker image has a newer stable version available",
|
|
[
|
|
"project",
|
|
"service",
|
|
"image",
|
|
"installed_version",
|
|
"available_version",
|
|
"registry",
|
|
],
|
|
registry=registry,
|
|
)
|
|
|
|
|
|
# ============================================================
|
|
# Version parsing
|
|
# ============================================================
|
|
|
|
def parse_version(tag):
|
|
"""
|
|
Extrait une version numérique située au début du tag.
|
|
"""
|
|
|
|
if not tag:
|
|
return None
|
|
|
|
tag = tag.strip()
|
|
|
|
match = re.match(
|
|
r"^v?(\d+(?:[._-]\d+)*)(?![A-Za-z0-9])",
|
|
tag,
|
|
)
|
|
|
|
if not match:
|
|
return None
|
|
|
|
value = match.group(1)
|
|
|
|
numbers = [
|
|
int(x)
|
|
for x in re.split(
|
|
r"[._-]",
|
|
value,
|
|
)
|
|
]
|
|
|
|
if numbers[0] > 999:
|
|
return None
|
|
|
|
return tuple(numbers)
|
|
|
|
|
|
def get_tag_suffix(tag):
|
|
"""
|
|
Retourne ce qui se trouve après la version numérique.
|
|
"""
|
|
|
|
if not tag:
|
|
return ""
|
|
|
|
match = re.match(
|
|
r"^v?(\d+(?:[._-]\d+)*)(?![A-Za-z0-9])(.*)$",
|
|
tag,
|
|
)
|
|
|
|
if not match:
|
|
return ""
|
|
|
|
suffix = match.group(2)
|
|
|
|
suffix = suffix.lstrip(
|
|
"._-"
|
|
)
|
|
|
|
return suffix
|
|
|
|
|
|
def get_build_variant(tag):
|
|
"""
|
|
Identifie les variantes de distribution/build.
|
|
"""
|
|
|
|
suffix = get_tag_suffix(tag)
|
|
|
|
if not suffix:
|
|
return ""
|
|
|
|
known_variants = (
|
|
"alpine",
|
|
"slim",
|
|
"bookworm",
|
|
"bullseye",
|
|
"buster",
|
|
"stretch",
|
|
"trixie",
|
|
"jammy",
|
|
"noble",
|
|
"focal",
|
|
"debian",
|
|
"ubuntu",
|
|
"ubi",
|
|
"distroless",
|
|
)
|
|
|
|
if suffix in known_variants:
|
|
return suffix
|
|
|
|
match = re.match(
|
|
r"^(alpine|bookworm|bullseye|buster|trixie)"
|
|
r"\d+(?:\.\d+)*$",
|
|
suffix,
|
|
)
|
|
|
|
if match:
|
|
return match.group(1)
|
|
|
|
match = re.match(
|
|
r"^(debian|ubuntu|alpine|bookworm|bullseye|trixie)"
|
|
r"(?:[-_]\d+(?:\.\d+)*)?$",
|
|
suffix,
|
|
)
|
|
|
|
if match:
|
|
return match.group(1)
|
|
|
|
return ""
|
|
|
|
|
|
# ============================================================
|
|
# Stable tag filtering
|
|
# ============================================================
|
|
|
|
def is_stable_tag(tag):
|
|
"""
|
|
Détermine si un tag ressemble à une release stable.
|
|
"""
|
|
|
|
if not tag:
|
|
return False
|
|
|
|
lower = tag.lower()
|
|
|
|
if lower in (
|
|
"latest",
|
|
"production",
|
|
"stable",
|
|
"edge",
|
|
"nightly",
|
|
"snapshot",
|
|
"dev",
|
|
"development",
|
|
"testing",
|
|
"test",
|
|
):
|
|
return False
|
|
|
|
if re.search(
|
|
r"(^|[-_.])"
|
|
r"(pr|pull|preview|dev|development|nightly|"
|
|
r"snapshot|master|main|branch|commit|sha)"
|
|
r"([-. _]|$)",
|
|
lower,
|
|
):
|
|
return False
|
|
|
|
if re.search(
|
|
r"(^|[-_.])"
|
|
r"(rc|alpha|beta|pre|preview|canary|"
|
|
r"unstable|experimental)"
|
|
r"(?:[-_.]|\d|$)",
|
|
lower,
|
|
):
|
|
return False
|
|
|
|
if re.search(
|
|
r"(^|[-_.])"
|
|
r"(amd64|x86_64|arm64|aarch64|armv7|armv6|"
|
|
r"386|ppc64le|s390x|riscv64|"
|
|
r"windows|win32|linux|darwin|"
|
|
r"freebsd|solaris)"
|
|
r"([-. _]|$)",
|
|
lower,
|
|
):
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
# ============================================================
|
|
# Docker image
|
|
# ============================================================
|
|
|
|
def split_image(image):
|
|
"""
|
|
Sépare registry / repository / tag.
|
|
"""
|
|
|
|
image = image.strip()
|
|
|
|
registry_name = "docker.io"
|
|
|
|
parts = image.split("/")
|
|
|
|
if len(parts) > 1 and (
|
|
"." in parts[0]
|
|
or ":" in parts[0]
|
|
or parts[0] == "localhost"
|
|
):
|
|
registry_name = parts[0]
|
|
parts = parts[1:]
|
|
|
|
if len(parts) == 1:
|
|
repository = (
|
|
"library/" + parts[0]
|
|
)
|
|
else:
|
|
repository = "/".join(parts)
|
|
|
|
tag = None
|
|
|
|
last = repository.rsplit(
|
|
"/",
|
|
1,
|
|
)[-1]
|
|
|
|
if ":" in last:
|
|
repository, tag = repository.rsplit(
|
|
":",
|
|
1,
|
|
)
|
|
|
|
if "@" in repository:
|
|
repository, digest = repository.split(
|
|
"@",
|
|
1,
|
|
)
|
|
|
|
tag = digest
|
|
|
|
return (
|
|
registry_name,
|
|
repository,
|
|
tag,
|
|
)
|
|
|
|
|
|
# ============================================================
|
|
# Registry token cache
|
|
# ============================================================
|
|
|
|
def get_cached_token(
|
|
registry_name,
|
|
repository,
|
|
):
|
|
"""
|
|
Retourne un token encore valide depuis le cache.
|
|
|
|
On garde une marge de sécurité de 60 secondes avant
|
|
l'expiration réelle du token.
|
|
"""
|
|
|
|
cache_key = (
|
|
registry_name,
|
|
repository,
|
|
)
|
|
|
|
cached = registry_tokens.get(
|
|
cache_key
|
|
)
|
|
|
|
if not cached:
|
|
return None
|
|
|
|
expires_at = cached.get(
|
|
"expires_at",
|
|
0,
|
|
)
|
|
|
|
if time.time() >= expires_at - 60:
|
|
|
|
registry_tokens.pop(
|
|
cache_key,
|
|
None,
|
|
)
|
|
|
|
print(
|
|
f" Registry : token expiré pour "
|
|
f"{registry_name}/{repository}"
|
|
)
|
|
|
|
return None
|
|
|
|
return cached.get(
|
|
"token"
|
|
)
|
|
|
|
|
|
def invalidate_registry_token(
|
|
registry_name,
|
|
repository,
|
|
):
|
|
"""
|
|
Supprime explicitement un token du cache.
|
|
"""
|
|
|
|
cache_key = (
|
|
registry_name,
|
|
repository,
|
|
)
|
|
|
|
if cache_key in registry_tokens:
|
|
|
|
registry_tokens.pop(
|
|
cache_key,
|
|
None,
|
|
)
|
|
|
|
print(
|
|
f" Registry : token invalidé pour "
|
|
f"{registry_name}/{repository}"
|
|
)
|
|
|
|
|
|
def store_registry_token(
|
|
registry_name,
|
|
repository,
|
|
token,
|
|
expires_in,
|
|
):
|
|
"""
|
|
Stocke un token avec sa date d'expiration.
|
|
|
|
expires_in est généralement fourni par le serveur
|
|
d'authentification.
|
|
|
|
Par sécurité, on limite également la durée maximale
|
|
du cache à 24 heures.
|
|
"""
|
|
|
|
try:
|
|
expires_in = int(
|
|
expires_in
|
|
)
|
|
except (
|
|
TypeError,
|
|
ValueError,
|
|
):
|
|
expires_in = 300
|
|
|
|
expires_in = max(
|
|
60,
|
|
min(
|
|
expires_in,
|
|
24 * 3600,
|
|
),
|
|
)
|
|
|
|
cache_key = (
|
|
registry_name,
|
|
repository,
|
|
)
|
|
|
|
registry_tokens[
|
|
cache_key
|
|
] = {
|
|
"token": token,
|
|
"expires_at": (
|
|
time.time()
|
|
+ expires_in
|
|
),
|
|
}
|
|
|
|
|
|
# ============================================================
|
|
# Docker Hub authentication
|
|
# ============================================================
|
|
|
|
def get_dockerhub_token(
|
|
repository,
|
|
force_refresh=False,
|
|
):
|
|
"""
|
|
Obtient un token Docker Hub.
|
|
|
|
Le token est conservé jusqu'à son expiration.
|
|
|
|
force_refresh=True permet de forcer une nouvelle
|
|
authentification, notamment après un HTTP 401 du Registry.
|
|
"""
|
|
|
|
if not force_refresh:
|
|
|
|
token = get_cached_token(
|
|
"docker.io",
|
|
repository,
|
|
)
|
|
|
|
if token:
|
|
return token
|
|
|
|
else:
|
|
|
|
invalidate_registry_token(
|
|
"docker.io",
|
|
repository,
|
|
)
|
|
|
|
url = (
|
|
"https://auth.docker.io/token"
|
|
"?service=registry.docker.io"
|
|
f"&scope=repository:{quote(repository, safe='')}:pull"
|
|
)
|
|
|
|
print(
|
|
f" Registry : nouveau token Docker Hub pour "
|
|
f"{repository}"
|
|
)
|
|
|
|
response = session.get(
|
|
url,
|
|
timeout=30,
|
|
)
|
|
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
token = data.get(
|
|
"token"
|
|
)
|
|
|
|
if not token:
|
|
raise RuntimeError(
|
|
"Docker Hub : token absent de la réponse"
|
|
)
|
|
|
|
expires_in = data.get(
|
|
"expires_in",
|
|
300,
|
|
)
|
|
|
|
store_registry_token(
|
|
"docker.io",
|
|
repository,
|
|
token,
|
|
expires_in,
|
|
)
|
|
|
|
return token
|
|
|
|
|
|
# ============================================================
|
|
# GHCR authentication
|
|
# ============================================================
|
|
|
|
def get_ghcr_token(
|
|
repository,
|
|
force_refresh=False,
|
|
):
|
|
"""
|
|
Obtient un token GHCR.
|
|
|
|
Même mécanisme de cache que Docker Hub.
|
|
"""
|
|
|
|
if not force_refresh:
|
|
|
|
token = get_cached_token(
|
|
"ghcr.io",
|
|
repository,
|
|
)
|
|
|
|
if token:
|
|
return token
|
|
|
|
else:
|
|
|
|
invalidate_registry_token(
|
|
"ghcr.io",
|
|
repository,
|
|
)
|
|
|
|
url = (
|
|
"https://ghcr.io/token"
|
|
f"?scope=repository:{quote(repository, safe='')}:pull"
|
|
)
|
|
|
|
print(
|
|
f" Registry : nouveau token GHCR pour "
|
|
f"{repository}"
|
|
)
|
|
|
|
response = session.get(
|
|
url,
|
|
timeout=30,
|
|
)
|
|
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
token = data.get(
|
|
"token"
|
|
)
|
|
|
|
if not token:
|
|
raise RuntimeError(
|
|
"GHCR : token absent de la réponse"
|
|
)
|
|
|
|
expires_in = data.get(
|
|
"expires_in",
|
|
300,
|
|
)
|
|
|
|
store_registry_token(
|
|
"ghcr.io",
|
|
repository,
|
|
token,
|
|
expires_in,
|
|
)
|
|
|
|
return token
|
|
|
|
|
|
# ============================================================
|
|
# Registry tags
|
|
# ============================================================
|
|
|
|
def get_registry_tags(
|
|
registry_name,
|
|
repository,
|
|
):
|
|
"""
|
|
Récupère TOUS les tags du registry.
|
|
|
|
Docker Hub :
|
|
registry-1.docker.io
|
|
|
|
GHCR :
|
|
ghcr.io
|
|
|
|
La pagination Link peut être absolue ou relative.
|
|
|
|
En cas de HTTP 401 sur un registry authentifié :
|
|
1. le token est supprimé ;
|
|
2. un nouveau token est demandé ;
|
|
3. la requête est rejouée une seule fois.
|
|
"""
|
|
|
|
cache_key = (
|
|
registry_name,
|
|
repository,
|
|
)
|
|
|
|
cached = registry_cache.get(
|
|
cache_key
|
|
)
|
|
|
|
if cached:
|
|
|
|
timestamp, tags = cached
|
|
|
|
if (
|
|
time.time() - timestamp
|
|
< REGISTRY_CACHE_TTL
|
|
):
|
|
|
|
print(
|
|
f" Registry : cache pour "
|
|
f"{registry_name}/{repository}"
|
|
)
|
|
|
|
return tags
|
|
|
|
print(
|
|
f" Registry : récupération des tags de "
|
|
f"{registry_name}/{repository}"
|
|
)
|
|
|
|
if registry_name == "docker.io":
|
|
|
|
base_url = (
|
|
DOCKER_HUB_REGISTRY_API
|
|
)
|
|
|
|
token = get_dockerhub_token(
|
|
repository
|
|
)
|
|
|
|
headers = {
|
|
"Authorization": (
|
|
f"Bearer {token}"
|
|
),
|
|
}
|
|
|
|
authenticated = True
|
|
|
|
elif registry_name == "ghcr.io":
|
|
|
|
base_url = "https://ghcr.io"
|
|
|
|
token = get_ghcr_token(
|
|
repository
|
|
)
|
|
|
|
headers = {
|
|
"Authorization": (
|
|
f"Bearer {token}"
|
|
),
|
|
}
|
|
|
|
authenticated = True
|
|
|
|
else:
|
|
|
|
base_url = (
|
|
f"https://{registry_name}"
|
|
)
|
|
|
|
headers = {}
|
|
|
|
authenticated = False
|
|
|
|
url = (
|
|
f"{base_url}/v2/"
|
|
f"{repository}/tags/list?n=100"
|
|
)
|
|
|
|
tags = []
|
|
|
|
# --------------------------------------------------------
|
|
# Une seule tentative normale + un éventuel renouvellement
|
|
# du token après 401.
|
|
# --------------------------------------------------------
|
|
|
|
token_refresh_attempted = False
|
|
|
|
while url:
|
|
|
|
response = session.get(
|
|
url,
|
|
headers=headers,
|
|
timeout=60,
|
|
)
|
|
|
|
# ----------------------------------------------------
|
|
# Token refusé.
|
|
#
|
|
# On ne boucle surtout pas indéfiniment :
|
|
# un seul renouvellement est tenté.
|
|
# ----------------------------------------------------
|
|
|
|
if (
|
|
response.status_code == 401
|
|
and authenticated
|
|
and not token_refresh_attempted
|
|
):
|
|
|
|
token_refresh_attempted = True
|
|
|
|
print(
|
|
f" Registry : HTTP 401 pour "
|
|
f"{registry_name}/{repository}"
|
|
)
|
|
|
|
print(
|
|
f" Registry : renouvellement du token pour "
|
|
f"{registry_name}/{repository}"
|
|
)
|
|
|
|
if registry_name == "docker.io":
|
|
|
|
token = get_dockerhub_token(
|
|
repository,
|
|
force_refresh=True,
|
|
)
|
|
|
|
else:
|
|
|
|
token = get_ghcr_token(
|
|
repository,
|
|
force_refresh=True,
|
|
)
|
|
|
|
headers = {
|
|
"Authorization": (
|
|
f"Bearer {token}"
|
|
),
|
|
}
|
|
|
|
# On rejoue exactement la même requête.
|
|
continue
|
|
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
tags.extend(
|
|
data.get(
|
|
"tags",
|
|
[],
|
|
)
|
|
)
|
|
|
|
next_url = None
|
|
|
|
link = response.headers.get(
|
|
"Link"
|
|
)
|
|
|
|
if link:
|
|
|
|
match = re.search(
|
|
r"<([^>]+)>;\s*rel=\"next\"",
|
|
link,
|
|
)
|
|
|
|
if match:
|
|
|
|
next_url = match.group(1)
|
|
|
|
if next_url.startswith("/"):
|
|
next_url = (
|
|
base_url
|
|
+ next_url
|
|
)
|
|
|
|
url = next_url
|
|
|
|
tags = list(
|
|
dict.fromkeys(tags)
|
|
)
|
|
|
|
registry_cache[
|
|
cache_key
|
|
] = (
|
|
time.time(),
|
|
tags,
|
|
)
|
|
|
|
print(
|
|
f" Registry : {len(tags)} tags pour "
|
|
f"{registry_name}/{repository}"
|
|
)
|
|
|
|
return tags
|
|
|
|
|
|
# ============================================================
|
|
# Bitnami
|
|
# ============================================================
|
|
|
|
def source_version_bitnami(
|
|
repository,
|
|
):
|
|
"""
|
|
Bitnami ne fournit pas toujours une liste de tags
|
|
exploitable via Registry.
|
|
|
|
On cherche les releases dans les PR GitHub.
|
|
"""
|
|
|
|
image_name = repository.rsplit(
|
|
"/",
|
|
1,
|
|
)[-1]
|
|
|
|
print(
|
|
f" GitHub Bitnami : recherche des versions "
|
|
f"de {repository}"
|
|
)
|
|
|
|
url = (
|
|
"https://api.github.com/search/issues"
|
|
)
|
|
|
|
query = (
|
|
f"repo:bitnami/containers "
|
|
f'"[bitnami/{image_name}] Release" '
|
|
f"is:pr"
|
|
)
|
|
|
|
headers = {
|
|
"Accept": (
|
|
"application/vnd.github+json"
|
|
),
|
|
"X-GitHub-Api-Version": "2022-11-28",
|
|
"User-Agent": (
|
|
"KAZ-docker-inventaire/1.0"
|
|
),
|
|
}
|
|
|
|
response = session.get(
|
|
url,
|
|
params={
|
|
"q": query,
|
|
"per_page": 100,
|
|
},
|
|
headers=headers,
|
|
timeout=60,
|
|
)
|
|
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
versions = []
|
|
|
|
for item in data.get(
|
|
"items",
|
|
[],
|
|
):
|
|
|
|
title = item.get(
|
|
"title",
|
|
"",
|
|
)
|
|
|
|
match = re.search(
|
|
rf"\[bitnami/{re.escape(image_name)}\]"
|
|
r"\s+Release\s+([0-9][^ ]*)",
|
|
title,
|
|
)
|
|
|
|
if not match:
|
|
continue
|
|
|
|
full_version = match.group(1)
|
|
|
|
version = parse_version(
|
|
full_version
|
|
)
|
|
|
|
if not version:
|
|
continue
|
|
|
|
if not is_stable_tag(
|
|
full_version
|
|
):
|
|
continue
|
|
|
|
numeric = ".".join(
|
|
str(x)
|
|
for x in version
|
|
)
|
|
|
|
versions.append(
|
|
(
|
|
version,
|
|
numeric,
|
|
)
|
|
)
|
|
|
|
if not versions:
|
|
return None
|
|
|
|
versions.sort(
|
|
key=lambda item: item[0]
|
|
)
|
|
|
|
return versions[-1][1]
|
|
|
|
|
|
# ============================================================
|
|
# Source version
|
|
# ============================================================
|
|
|
|
def source_version_for(
|
|
image,
|
|
):
|
|
"""
|
|
Retourne la dernière version stable disponible.
|
|
"""
|
|
|
|
(
|
|
registry_name,
|
|
repository,
|
|
current_tag,
|
|
) = split_image(image)
|
|
|
|
if not current_tag:
|
|
return None
|
|
|
|
if current_tag in (
|
|
"latest",
|
|
"production",
|
|
):
|
|
return None
|
|
|
|
current_version = parse_version(
|
|
current_tag
|
|
)
|
|
|
|
if not current_version:
|
|
return None
|
|
|
|
current_variant = get_build_variant(
|
|
current_tag
|
|
)
|
|
|
|
# --------------------------------------------------------
|
|
# Bitnami
|
|
# --------------------------------------------------------
|
|
|
|
if (
|
|
registry_name == "docker.io"
|
|
and repository.startswith(
|
|
"bitnami/"
|
|
)
|
|
):
|
|
|
|
try:
|
|
|
|
version = (
|
|
source_version_bitnami(
|
|
repository
|
|
)
|
|
)
|
|
|
|
if version:
|
|
|
|
print(
|
|
f" {repository} -> {version}"
|
|
)
|
|
|
|
return version
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f" Bitnami erreur: {exc}"
|
|
)
|
|
|
|
return None
|
|
|
|
# --------------------------------------------------------
|
|
# Registry
|
|
# --------------------------------------------------------
|
|
|
|
try:
|
|
|
|
tags = get_registry_tags(
|
|
registry_name,
|
|
repository,
|
|
)
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f" Registry erreur: {exc}"
|
|
)
|
|
|
|
return None
|
|
|
|
candidates = []
|
|
|
|
for tag in tags:
|
|
|
|
if not tag:
|
|
continue
|
|
|
|
if tag in (
|
|
"latest",
|
|
"production",
|
|
):
|
|
continue
|
|
|
|
if not is_stable_tag(
|
|
tag
|
|
):
|
|
continue
|
|
|
|
version = parse_version(
|
|
tag
|
|
)
|
|
|
|
if not version:
|
|
continue
|
|
|
|
variant = get_build_variant(
|
|
tag
|
|
)
|
|
|
|
if current_variant:
|
|
|
|
if variant != current_variant:
|
|
continue
|
|
|
|
exact = (
|
|
1
|
|
if tag.startswith("v")
|
|
else 0
|
|
)
|
|
|
|
no_suffix = (
|
|
1
|
|
if get_tag_suffix(tag) == ""
|
|
else 0
|
|
)
|
|
|
|
candidates.append(
|
|
(
|
|
version,
|
|
no_suffix,
|
|
exact,
|
|
tag,
|
|
)
|
|
)
|
|
|
|
if not candidates:
|
|
|
|
print(
|
|
f" Source : aucune version compatible pour "
|
|
f"{image}"
|
|
)
|
|
|
|
return None
|
|
|
|
candidates.sort(
|
|
key=lambda item: (
|
|
item[0],
|
|
item[1],
|
|
item[2],
|
|
)
|
|
)
|
|
|
|
source_version = candidates[-1][3]
|
|
|
|
print(
|
|
f" Source : {image} -> "
|
|
f"{source_version}"
|
|
)
|
|
|
|
return source_version
|
|
|
|
|
|
# ============================================================
|
|
# Git archive
|
|
# ============================================================
|
|
|
|
def git_archive():
|
|
|
|
print(
|
|
f"Git : récupération de {GIT_URL}"
|
|
)
|
|
|
|
# Nettoyage des archives temporaires précédentes
|
|
for path in glob.glob("/tmp/docker-inventaire-*"):
|
|
if os.path.isdir(path):
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
|
|
tempdir = tempfile.mkdtemp(
|
|
prefix="docker-inventaire-"
|
|
)
|
|
|
|
archive_path = os.path.join(
|
|
tempdir,
|
|
"dockers.tar",
|
|
)
|
|
|
|
command = [
|
|
"git",
|
|
"archive",
|
|
"--remote=" + GIT_URL,
|
|
GIT_REF,
|
|
"dockers",
|
|
]
|
|
|
|
with open(
|
|
archive_path,
|
|
"wb",
|
|
) as archive:
|
|
|
|
process = subprocess.run(
|
|
command,
|
|
stdout=archive,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
|
|
if process.returncode != 0:
|
|
|
|
raise RuntimeError(
|
|
"git archive a échoué : "
|
|
+ process.stderr.decode(
|
|
errors="replace"
|
|
)
|
|
)
|
|
|
|
extract_dir = os.path.join(
|
|
tempdir,
|
|
"src",
|
|
)
|
|
|
|
os.makedirs(
|
|
extract_dir,
|
|
exist_ok=True,
|
|
)
|
|
|
|
with tarfile.open(
|
|
archive_path,
|
|
"r",
|
|
) as tar:
|
|
|
|
for member in tar.getmembers():
|
|
|
|
if (
|
|
member.issym()
|
|
or member.islnk()
|
|
):
|
|
continue
|
|
|
|
tar.extract(
|
|
member,
|
|
extract_dir,
|
|
filter="data",
|
|
)
|
|
|
|
print(
|
|
"Git : archive extraite"
|
|
)
|
|
|
|
return extract_dir
|
|
|
|
|
|
# ============================================================
|
|
# Compose parser
|
|
# ============================================================
|
|
|
|
def parse_compose_file(
|
|
path,
|
|
):
|
|
"""
|
|
Parse le compose comme du texte.
|
|
"""
|
|
|
|
result = []
|
|
|
|
try:
|
|
|
|
with open(
|
|
path,
|
|
"r",
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
) as file:
|
|
|
|
lines = file.readlines()
|
|
|
|
except Exception:
|
|
|
|
return result
|
|
|
|
current_service = None
|
|
service_indent = None
|
|
|
|
for line in lines:
|
|
|
|
if not line.strip():
|
|
continue
|
|
|
|
if line.lstrip().startswith("#"):
|
|
continue
|
|
|
|
indent = (
|
|
len(line)
|
|
- len(line.lstrip())
|
|
)
|
|
|
|
stripped = line.strip()
|
|
|
|
match_service = re.match(
|
|
r"^([A-Za-z0-9_.-]+):\s*$",
|
|
stripped,
|
|
)
|
|
|
|
if (
|
|
match_service
|
|
and indent >= 2
|
|
):
|
|
|
|
current_service = (
|
|
match_service.group(1)
|
|
)
|
|
|
|
service_indent = indent
|
|
|
|
continue
|
|
|
|
if (
|
|
current_service
|
|
and service_indent is not None
|
|
and indent > service_indent
|
|
):
|
|
|
|
match_image = re.match(
|
|
r"^image:\s*['\"]?([^'\"\s]+)",
|
|
stripped,
|
|
)
|
|
|
|
if match_image:
|
|
|
|
image = match_image.group(
|
|
1
|
|
)
|
|
|
|
if (
|
|
image.endswith(
|
|
":latest"
|
|
)
|
|
or image.endswith(
|
|
":production"
|
|
)
|
|
):
|
|
continue
|
|
|
|
if ":" not in image:
|
|
continue
|
|
|
|
(
|
|
registry_name,
|
|
repository,
|
|
tag,
|
|
) = split_image(image)
|
|
|
|
if not tag:
|
|
continue
|
|
|
|
if not parse_version(tag):
|
|
continue
|
|
|
|
result.append(
|
|
(
|
|
current_service,
|
|
image,
|
|
)
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
# ============================================================
|
|
# Inventaire
|
|
# ============================================================
|
|
|
|
def inventaire():
|
|
|
|
print(
|
|
"=" * 50
|
|
)
|
|
|
|
print(
|
|
"Début inventaire"
|
|
)
|
|
|
|
# --------------------------------------------------------
|
|
# Nettoyage des anciennes métriques
|
|
# --------------------------------------------------------
|
|
|
|
git_metric._metrics.clear()
|
|
source_metric._metrics.clear()
|
|
version_metric._metrics.clear()
|
|
update_metric._metrics.clear()
|
|
|
|
# --------------------------------------------------------
|
|
# Git
|
|
# --------------------------------------------------------
|
|
|
|
try:
|
|
|
|
source_dir = git_archive()
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f"Git erreur : {exc}"
|
|
)
|
|
|
|
return
|
|
|
|
compose_files = []
|
|
|
|
for root, dirs, files in os.walk(
|
|
os.path.join(
|
|
source_dir,
|
|
"dockers",
|
|
)
|
|
):
|
|
|
|
for filename in files:
|
|
|
|
if filename in (
|
|
"docker-compose.yml",
|
|
"docker-compose.yaml",
|
|
"docker-compose.yml.dist",
|
|
"docker-compose.yaml.dist",
|
|
"docker-compose.tmpl.yml.dist",
|
|
"docker-compose.tmpl.yaml.dist",
|
|
):
|
|
|
|
compose_files.append(
|
|
os.path.join(
|
|
root,
|
|
filename,
|
|
)
|
|
)
|
|
|
|
print(
|
|
f"Git : {len(compose_files)} fichiers compose"
|
|
)
|
|
|
|
# --------------------------------------------------------
|
|
# Images
|
|
# --------------------------------------------------------
|
|
|
|
images = {}
|
|
|
|
for compose_file in compose_files:
|
|
|
|
relative = os.path.relpath(
|
|
compose_file,
|
|
os.path.join(
|
|
source_dir,
|
|
"dockers",
|
|
),
|
|
)
|
|
|
|
project = relative.split(
|
|
os.sep,
|
|
1,
|
|
)[0]
|
|
|
|
entries = parse_compose_file(
|
|
compose_file
|
|
)
|
|
|
|
for service, image in entries:
|
|
|
|
(
|
|
registry_name,
|
|
repository,
|
|
tag,
|
|
) = split_image(image)
|
|
|
|
if not tag:
|
|
continue
|
|
|
|
git_metric.labels(
|
|
project=project,
|
|
service=service,
|
|
image=image,
|
|
version=tag,
|
|
).set(1)
|
|
|
|
images.setdefault(
|
|
image,
|
|
[],
|
|
).append(
|
|
{
|
|
"project": project,
|
|
"service": service,
|
|
"registry": registry_name,
|
|
"installed_version": tag,
|
|
}
|
|
)
|
|
|
|
# --------------------------------------------------------
|
|
# Suppression des doublons
|
|
# --------------------------------------------------------
|
|
|
|
for image in images:
|
|
|
|
unique_entries = []
|
|
seen = set()
|
|
|
|
for entry in images[image]:
|
|
|
|
key = (
|
|
entry["project"],
|
|
entry["service"],
|
|
entry["registry"],
|
|
entry["installed_version"],
|
|
)
|
|
|
|
if key in seen:
|
|
continue
|
|
|
|
seen.add(key)
|
|
|
|
unique_entries.append(
|
|
entry
|
|
)
|
|
|
|
images[image] = unique_entries
|
|
|
|
print(
|
|
f"Git : {len(images)} images distinctes"
|
|
)
|
|
|
|
# --------------------------------------------------------
|
|
# Source
|
|
# --------------------------------------------------------
|
|
|
|
print(
|
|
"Recherche des versions Source..."
|
|
)
|
|
|
|
for image in sorted(
|
|
images
|
|
):
|
|
|
|
source_version = None
|
|
|
|
try:
|
|
|
|
(
|
|
registry_name,
|
|
repository,
|
|
current_tag,
|
|
) = split_image(image)
|
|
|
|
source_version = (
|
|
source_version_for(
|
|
image
|
|
)
|
|
)
|
|
|
|
if source_version:
|
|
|
|
source_metric.labels(
|
|
image=image,
|
|
version=source_version,
|
|
registry=registry_name,
|
|
).set(1)
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f" Erreur Source {image}: "
|
|
f"{exc}"
|
|
)
|
|
|
|
# ----------------------------------------------------
|
|
# Métrique complète pour Grafana
|
|
# ----------------------------------------------------
|
|
|
|
available_version = (
|
|
source_version
|
|
if source_version
|
|
else ""
|
|
)
|
|
|
|
for entry in images[image]:
|
|
|
|
installed_version = entry[
|
|
"installed_version"
|
|
]
|
|
|
|
version_metric.labels(
|
|
project=entry["project"],
|
|
service=entry["service"],
|
|
image=image,
|
|
installed_version=installed_version,
|
|
available_version=available_version,
|
|
registry=entry["registry"],
|
|
).set(1)
|
|
|
|
update_metric.labels(
|
|
project=entry["project"],
|
|
service=entry["service"],
|
|
image=image,
|
|
installed_version=installed_version,
|
|
available_version=available_version,
|
|
registry=entry["registry"],
|
|
).set(
|
|
1 if (
|
|
available_version
|
|
and installed_version != available_version
|
|
) else 0
|
|
)
|
|
|
|
print(
|
|
"Fin inventaire"
|
|
)
|
|
|
|
print(
|
|
"=" * 50
|
|
)
|
|
|
|
|
|
# ============================================================
|
|
# HTTP Prometheus
|
|
# ============================================================
|
|
|
|
class MetricsHandler(
|
|
BaseHTTPRequestHandler
|
|
):
|
|
|
|
def do_GET(self):
|
|
|
|
if self.path != "/metrics":
|
|
|
|
self.send_response(
|
|
404
|
|
)
|
|
|
|
self.end_headers()
|
|
|
|
return
|
|
|
|
output = generate_latest(
|
|
registry
|
|
)
|
|
|
|
self.send_response(
|
|
200
|
|
)
|
|
|
|
self.send_header(
|
|
"Content-Type",
|
|
"text/plain; version=0.0.4",
|
|
)
|
|
|
|
self.send_header(
|
|
"Content-Length",
|
|
str(len(output)),
|
|
)
|
|
|
|
self.end_headers()
|
|
|
|
self.wfile.write(
|
|
output
|
|
)
|
|
|
|
def log_message(
|
|
self,
|
|
format,
|
|
*args,
|
|
):
|
|
return
|
|
|
|
|
|
# ============================================================
|
|
# Main
|
|
# ============================================================
|
|
|
|
def main():
|
|
|
|
print(
|
|
"Démarrage docker-inventaire "
|
|
f"sur le port {PORT}"
|
|
)
|
|
|
|
server = HTTPServer(
|
|
(
|
|
"0.0.0.0",
|
|
PORT,
|
|
),
|
|
MetricsHandler,
|
|
)
|
|
|
|
def run_inventory():
|
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
inventaire()
|
|
|
|
except Exception as exc:
|
|
|
|
print(
|
|
f"ERREUR inventaire : {exc}"
|
|
)
|
|
|
|
time.sleep(
|
|
INVENTORY_INTERVAL
|
|
)
|
|
|
|
thread = threading.Thread(
|
|
target=run_inventory,
|
|
daemon=True,
|
|
)
|
|
|
|
thread.start()
|
|
|
|
server.serve_forever()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|