NewWebConfigurator/src/log_manager.py

"""
log_manager.py — Log File Management Module

Provides log file listing, secure download archive creation,
and automatic compression for /opt/log/dpworldapp.
"""

import os
import re
import tarfile
import tempfile
import time
import shutil
import threading
import logging

LOG_DIR = os.environ.get("LOG_DIR", "/opt/log/dpworldapp")
# v1.4.6.6: systemd ProtectSystem=strict + ReadWritePaths 화이트리스트가 /opt/log 부재로
# `/opt/log/log-download-*` mkdtemp 차단 (.56 라이브 OSError 30). PrivateTmp=yes 격리된
# tmpfs /tmp로 default 변경. 운영자가 다른 경로 원하면 LOG_DOWNLOAD_TEMP_PARENT env로 override.
_DOWNLOAD_TEMP_PARENT = os.environ.get("LOG_DOWNLOAD_TEMP_PARENT", "/tmp")
_DOWNLOAD_TEMP_PREFIX = "log-download-"
MAX_DOWNLOAD_SIZE = 100 * 1024 * 1024  # 100MB download limit
ALLOWED_EXTENSIONS = frozenset({'.log', '.tar.gz', '.gz', '.txt'})
# Uncompressed rotation suffixes produced by logrotate on devices without
# the auto-compress daemon — e.g. dpworldapp_2026-05-25.log.0 / .log.1
_ROTATION_SUFFIX_RE = re.compile(r'\.log\.\d+$')
COMPRESS_CHECK_INTERVAL = 5 * 60  # 5 minutes

# Log-management settings live in their own `log_config` board_config key —
# NOT in `device_config`. The legacy Java app overwrites device_config with
# its own model and silently drops these Python-only fields.
LOG_CONFIG_KEYS = (
    "log_auto_compress",
    "log_auto_cleanup",
    "log_cleanup_max_files",
    "log_cleanup_max_size_mb",
    # v1.4.6.7 C-1: Python 전용 compress threshold (Java schema 부재 → device_config에서 wipe 위험).
    # .56 라이브 monitoring Phase 1에서 device_config로 잘못 진입 노출 (state.js convertNestedToFlat이
    # 8개 log_* 키 모두 emit인데 LOG_CONFIG_KEYS가 4개만 분리 — 누락 2개).
    "log_compress_size_mb",
    "log_compress_age_days",
)
LOG_CONFIG_DEFAULTS = {
    "log_auto_compress": "off",
    "log_auto_cleanup": "off",
    "log_cleanup_max_files": 50,
    "log_cleanup_max_size_mb": 200,
    # v1.4.6.7 C-1: state.js convertNestedToFlat과 default 일치
    "log_compress_size_mb": 50,
    "log_compress_age_days": 7,
}

logger = logging.getLogger(__name__)

# Matches .log and rotated files like .log.0, .log.1, .log.2, etc.
_LOG_FILE_RE = re.compile(r'\.log(\.\d+)?$')
_compression_lock = threading.Lock()


class CompressionAlreadyRunning(RuntimeError):
    """Raised when a second compression request starts before the first finishes."""


def _is_log_file(name):
    """Check if filename is a log file (including rotated ones like .log.0, .log.1)."""
    return bool(_LOG_FILE_RE.search(name))


def _has_allowed_extension(name):
    """True if `name`'s extension is in ALLOWED_EXTENSIONS, or matches a
    logrotate numeric suffix like '.log.0' / '.log.1'."""
    if any(name.endswith(ext) for ext in ALLOWED_EXTENSIONS):
        return True
    return bool(_ROTATION_SUFFIX_RE.search(name))


# B12: sentinel returned by _resolve_log_path when the extension check fails.
# Callers use different wording for this case ("File extension not allowed" vs
# "Unsupported extension") so they format the message themselves.
_EXT_DENIED = object()


def _resolve_log_path(name):
    """Perform the 5-step security gate for a single filename.

    Returns (resolved_path, None) on success.
    Returns (None, _EXT_DENIED) if the extension is not whitelisted — callers
    must produce their own error message for this case.
    Returns (None, error_str) for all other failures.
    """
    if not isinstance(name, str) or not name.strip():
        return None, "Invalid filename: empty or non-string"
    if '..' in name or '/' in name or '\\' in name:
        return None, f"Invalid filename: {name}"
    if not _has_allowed_extension(name):
        return None, _EXT_DENIED
    filepath = os.path.realpath(os.path.join(LOG_DIR, name))
    if not filepath.startswith(os.path.realpath(LOG_DIR) + os.sep):
        return None, f"Invalid filename: {name}"
    if not os.path.isfile(filepath):
        return None, f"File not found: {name}"
    return filepath, None


def _make_tar_name(name, mtime):
    """
    Archive name for a log file: ``<original name>.<log-mtime>.tar.gz``.

    The full original name is kept (so ``.log`` and rotated ``.log.N`` are
    treated identically) and the log's mtime is embedded — making the name
    unique, so a log filename reused over time never collides with a prior
    archive. Example: ``dpworldapp_2026-05-21.log.20260521-164801.tar.gz``.
    """
    ts = time.strftime('%Y%m%d-%H%M%S', time.localtime(mtime))
    return f"{name}.{ts}.tar.gz"


def _verify_archive(tar_path, expected_name):
    """Return True if tar_path is a valid .tar.gz holding exactly expected_name."""
    try:
        with tarfile.open(tar_path, 'r:gz') as tar:
            members = tar.getmembers()
        return len(members) == 1 and members[0].name == expected_name
    except Exception:
        return False


def _safe_remove(path):
    """Remove a file, ignoring any error."""
    try:
        os.remove(path)
    except OSError:
        pass


def list_log_files():
    """
    List log files in LOG_DIR with metadata.
    Returns dict: { files: [...], total_size: int }
    """
    if not os.path.isdir(LOG_DIR):
        return {"files": [], "total_size": 0}

    files = []
    total_size = 0

    try:
        names = os.listdir(LOG_DIR)
    except OSError as e:
        raise RuntimeError(f"Failed to read log directory: {e}")

    for name in names:
        filepath = os.path.join(LOG_DIR, name)
        try:
            if not os.path.isfile(filepath):
                continue
            stat = os.stat(filepath)
        except OSError:
            # File disappeared between listdir and stat (compress daemon race).
            continue
        size = stat.st_size
        mtime = stat.st_mtime

        # Determine file type
        if name.endswith('.tar.gz'):
            ftype = 'tar.gz'
        elif name.endswith('.gz'):
            ftype = 'gz'
        elif _is_log_file(name):
            ftype = 'log'
        elif name.endswith('.txt'):
            ftype = 'txt'
        else:
            ftype = 'other'

        files.append({
            "name": name,
            "size": size,
            "modified": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(mtime)),
            "type": ftype,
        })
        total_size += size

    # Sort by modified descending (newest first)
    files.sort(key=lambda f: f["modified"], reverse=True)

    return {"files": files, "total_size": total_size}


def validate_filenames(filenames):
    """
    Validate a list of filenames for security.
    Blocks path traversal, absolute paths, and missing files.
    Returns (valid_paths, error_message).
    """
    if not filenames or not isinstance(filenames, list):
        return None, "No files specified"

    valid_paths = []

    for name in filenames:
        filepath, err = _resolve_log_path(name)
        if err is _EXT_DENIED:
            return None, f"File extension not allowed: {name}"
        if err is not None:
            return None, err
        valid_paths.append(filepath)

    # Check total size limit
    total = sum(os.path.getsize(p) for p in valid_paths)
    if total > MAX_DOWNLOAD_SIZE:
        return None, f"Total size exceeds {MAX_DOWNLOAD_SIZE // (1024*1024)}MB limit"

    return valid_paths, None


def create_download_archive(valid_paths):
    """
    Create a tar.gz archive on disk under _DOWNLOAD_TEMP_PARENT.
    Returns the absolute archive path. Caller is responsible for removing
    os.path.dirname(archive_path) after streaming.

    Per-file FileNotFoundError is logged and skipped (resilient to the
    auto-compress daemon renaming/deleting files during the request).

    v1.4.6.5 L-D: If ALL files vanish before being added, the workspace is
    cleaned up and FileNotFoundError is raised so the caller can return
    HTTP 404 instead of an empty tar.gz with HTTP 200.
    """
    os.makedirs(_DOWNLOAD_TEMP_PARENT, exist_ok=True)
    workspace = tempfile.mkdtemp(prefix=_DOWNLOAD_TEMP_PREFIX, dir=_DOWNLOAD_TEMP_PARENT)
    try:
        timestamp = time.strftime("%Y-%m-%d_%H%M%S")
        archive_path = os.path.join(workspace, f"logs_{timestamp}.tar.gz")
        added_count = 0
        with tarfile.open(archive_path, mode='w:gz') as tar:
            for filepath in valid_paths:
                arcname = os.path.basename(filepath)
                try:
                    tar.add(filepath, arcname=arcname)
                    added_count += 1
                except FileNotFoundError:
                    # Daemon may have renamed/deleted between validate and archive.
                    # Skip; client just gets a smaller archive.
                    continue
        if added_count == 0:
            shutil.rmtree(workspace, ignore_errors=True)
            raise FileNotFoundError(
                "All requested files disappeared between validation and archive"
            )
        return archive_path
    except Exception:
        shutil.rmtree(workspace, ignore_errors=True)
        raise


def sweep_stale_download_temp_dirs():
    """Remove leftover log-download temp dirs from prior server crashes."""
    if not os.path.isdir(_DOWNLOAD_TEMP_PARENT):
        return 0
    removed = 0
    for name in os.listdir(_DOWNLOAD_TEMP_PARENT):
        if not name.startswith(_DOWNLOAD_TEMP_PREFIX):
            continue
        path = os.path.join(_DOWNLOAD_TEMP_PARENT, name)
        if not os.path.isdir(path):
            continue
        try:
            shutil.rmtree(path)
            removed += 1
        except OSError:
            pass
    return removed


def get_active_log_file():
    """
    Return the filename of the most recently modified log file (the active one).
    DPWORLDAPP writes to one log file at a time; completed files should be compressed.
    Considers .log and rotated .log.0, .log.1, etc.
    Returns None if no log files exist.
    """
    if not os.path.isdir(LOG_DIR):
        return None

    latest_name = None
    latest_mtime = 0

    try:
        for name in os.listdir(LOG_DIR):
            if not _is_log_file(name):
                continue
            filepath = os.path.join(LOG_DIR, name)
            if not os.path.isfile(filepath):
                continue
            mtime = os.stat(filepath).st_mtime
            if mtime > latest_mtime:
                latest_mtime = mtime
                latest_name = name
    except OSError:
        pass

    return latest_name


def get_log_stats():
    """
    Return disk usage statistics for the log directory.
    Returns dict with compressed/uncompressed counts, sizes, active file, total size,
    and partition capacity (total/used/free bytes for the filesystem hosting LOG_DIR).
    """
    # Partition capacity — fall back to root if LOG_DIR is absent
    try:
        probe = LOG_DIR if os.path.isdir(LOG_DIR) else "/"
        usage = shutil.disk_usage(probe)
        partition_total = usage.total
        partition_used = usage.used
        partition_free = usage.free
    except (OSError, AttributeError):
        partition_total = partition_used = partition_free = 0

    if not os.path.isdir(LOG_DIR):
        return {
            "total_size": 0,
            "compressed_count": 0,
            "compressed_size": 0,
            "uncompressed_count": 0,
            "uncompressed_size": 0,
            "active_file": None,
            "partition_total": partition_total,
            "partition_used": partition_used,
            "partition_free": partition_free,
        }

    active = get_active_log_file()
    compressed_count = 0
    compressed_size = 0
    uncompressed_count = 0
    uncompressed_size = 0
    total_size = 0

    try:
        for name in os.listdir(LOG_DIR):
            filepath = os.path.join(LOG_DIR, name)
            if not os.path.isfile(filepath):
                continue
            size = os.stat(filepath).st_size
            total_size += size

            if name.endswith('.tar.gz') or name.endswith('.gz'):
                compressed_count += 1
                compressed_size += size
            else:
                uncompressed_count += 1
                uncompressed_size += size
    except OSError:
        pass

    return {
        "total_size": total_size,
        "compressed_count": compressed_count,
        "compressed_size": compressed_size,
        "uncompressed_count": uncompressed_count,
        "uncompressed_size": uncompressed_size,
        "active_file": active,
        "partition_total": partition_total,
        "partition_used": partition_used,
        "partition_free": partition_free,
    }


def _compress_non_active_logs():
    if not _compression_lock.acquire(blocking=False):
        raise CompressionAlreadyRunning("Compression already running")
    try:
        return _compress_non_active_logs_unlocked()
    finally:
        _compression_lock.release()


def _compress_non_active_logs_unlocked():
    """
    Compress every log file except the active (most recently modified) one.

    Handles ``.log`` and rotated ``.log.0``, ``.log.1``, etc. Each archive is
    named ``<logname>.<log-mtime>.tar.gz`` (see _make_tar_name) — unique, so a
    reused log filename never collides with a prior, possibly stale, archive.
    Archives are written to a ``.partial`` temp file and atomically renamed, so
    an interrupted run never leaves a corrupt archive that shadows the log.
    Returns the number of files compressed.
    """
    if not os.path.isdir(LOG_DIR):
        return 0

    try:
        names = os.listdir(LOG_DIR)
    except OSError as e:
        logger.warning(f"Error listing log directory: {e}")
        return 0

    # Clear leftover .partial archives from any previously interrupted run.
    for name in names:
        if name.endswith('.tar.gz.partial'):
            _safe_remove(os.path.join(LOG_DIR, name))

    active = get_active_log_file()
    compressed_count = 0

    for name in names:
        if not _is_log_file(name):
            continue

        # Skip the active file
        if name == active:
            continue

        filepath = os.path.join(LOG_DIR, name)
        if not os.path.isfile(filepath):
            continue

        try:
            mtime = os.stat(filepath).st_mtime
        except OSError:
            continue

        tar_name = _make_tar_name(name, mtime)
        tar_path = os.path.join(LOG_DIR, tar_name)

        # This exact version (same name + mtime) already archived?
        if os.path.exists(tar_path):
            if _verify_archive(tar_path, name):
                # Archive is good — drop the now-redundant original.
                _safe_remove(filepath)
                continue
            # Corrupt/partial archive at the final name — discard, recompress.
            _safe_remove(tar_path)

        tmp_path = tar_path + '.partial'
        try:
            with tarfile.open(tmp_path, 'w:gz') as tar:
                tar.add(filepath, arcname=name)

            if not _verify_archive(tmp_path, name):
                _safe_remove(tmp_path)
                logger.warning(f"Archive verification failed for {name}")
                continue

            os.rename(tmp_path, tar_path)   # atomic publish
            os.remove(filepath)
            compressed_count += 1
            logger.info(f"Compressed: {name} -> {tar_name}")

        except Exception as e:
            logger.warning(f"Failed to compress {name}: {e}")
            _safe_remove(tmp_path)

    return compressed_count


def _run_cleanup(max_files, max_size_mb):
    """
    Delete oldest compressed (.tar.gz / .gz) archives until both thresholds are met:
      - file count <= max_files
      - total compressed size <= max_size_mb (MB)
    Uncompressed and active log files are never touched.
    Returns the number of files deleted.
    """
    if not os.path.isdir(LOG_DIR):
        return 0

    # v1.11.8 B: defensive count floor — a corrupted/zero max_files from the DB
    # must never delete EVERY archive. Keep at least one. (The size threshold is
    # independent and can still bring the set down if the operator set it tiny.)
    try:
        max_files = max(1, int(max_files))
    except (TypeError, ValueError):
        max_files = 1

    max_size_bytes = max_size_mb * 1024 * 1024
    archives = []

    try:
        for name in os.listdir(LOG_DIR):
            if not (name.endswith('.tar.gz') or name.endswith('.gz')):
                continue
            filepath = os.path.join(LOG_DIR, name)
            if not os.path.isfile(filepath):
                continue
            try:
                stat = os.stat(filepath)
                archives.append((stat.st_mtime, stat.st_size, name, filepath))
            except OSError:
                continue
    except OSError as e:
        logger.warning(f"Cleanup: error listing log directory: {e}")
        return 0

    # Oldest first
    archives.sort(key=lambda a: a[0])

    deleted = 0
    total_size = sum(a[1] for a in archives)

    while len(archives) > 1 and (len(archives) > max_files or total_size > max_size_bytes):
        entry = archives.pop(0)
        mtime, size, name, path = entry
        try:
            os.remove(path)
            total_size -= size
            deleted += 1
            logger.info(
                f"Auto-cleanup: deleted {name} (mtime={mtime:.0f}, size={size})"
            )
        except OSError as e:
            # #24 fix: the file is still on disk — re-append it (so the count is
            # honest) and stop. Continuing would drop it from `archives`, shrinking
            # len(archives) toward the threshold and exiting the count-based loop
            # one early while the file remains. We can't make progress past an
            # un-removable oldest file, so break to avoid a no-progress spin.
            logger.warning(f"Cleanup: failed to delete {name}: {e}")
            archives.append(entry)
            break

    return deleted


def migrate_log_config(db):
    """
    Ensure the `log_config` board_config key exists.

    Log-management settings used to live in `device_config`, which the legacy
    Java app overwrites with its own model — silently dropping them. They now
    live in their own key. This seeds `log_config` once, from any values still
    present in `device_config`, otherwise from defaults. Idempotent.
    """
    try:
        if db.config_exists("log_config"):
            return
        device = db.get_config("device_config") or {}
        seeded = {
            key: device.get(key, LOG_CONFIG_DEFAULTS[key])
            for key in LOG_CONFIG_KEYS
        }
        # B11: clamp integer fields to the same hard floors enforced by
        # validate_log_config_hard — a legacy device_config with sub-floor
        # values (e.g. log_cleanup_max_size_mb=0) must not be seeded as-is.
        _seed_floors = {
            "log_cleanup_max_files": 5,
            "log_cleanup_max_size_mb": 50,
            "log_compress_size_mb": 1,
            "log_compress_age_days": 1,
        }
        for key, floor in _seed_floors.items():
            v = seeded.get(key)
            try:
                seeded[key] = max(floor, int(v))
            except (TypeError, ValueError):
                seeded[key] = LOG_CONFIG_DEFAULTS[key]
        db.save_config("log_config", seeded)
        logger.info("Migrated log-management settings into the log_config key")
    except Exception as e:
        logger.warning(f"log_config migration error: {e}")


def run_auto_compression(db):
    """
    Compress ALL non-active .log files if auto_compress is enabled.
    Reads settings from the log_config key in DB.
    """
    try:
        config = db.get_config("log_config")
        if not isinstance(config, dict) or not config:
            return

        # Check if auto compress is enabled
        auto_compress = config.get("log_auto_compress", "off")
        if auto_compress != "on":
            return

        try:
            count = _compress_non_active_logs()
            if count > 0:
                logger.info(f"Auto-compressed {count} file(s)")
        except CompressionAlreadyRunning:
            logger.info("Auto compression skipped: compression already running")

    except Exception as e:
        logger.warning(f"Auto compression error: {e}")


def run_auto_cleanup(db):
    """
    Delete oldest compressed log archives if log_auto_cleanup is enabled
    and either threshold (max_files or max_size_mb) is exceeded.
    Reads settings from the log_config key in DB. Uses safe defaults when keys missing.
    """
    try:
        config = db.get_config("log_config")
        if not isinstance(config, dict) or not config:
            return

        if config.get("log_auto_cleanup", "off") != "on":
            return

        try:
            max_files = int(config.get("log_cleanup_max_files", 50))
        except (TypeError, ValueError):
            max_files = 50
        try:
            max_size_mb = int(config.get("log_cleanup_max_size_mb", 200))
        except (TypeError, ValueError):
            max_size_mb = 200

        count = _run_cleanup(max_files, max_size_mb)
        if count > 0:
            logger.info(f"Auto-cleanup deleted {count} archive(s)")

    except Exception as e:
        logger.warning(f"Auto cleanup error: {e}")


def run_manual_compression():
    """
    Manually compress ALL non-active .log files.
    Always runs regardless of auto_compress setting.
    Returns dict with result info.
    """
    try:
        count = _compress_non_active_logs()
        return {
            "success": True,
            "compressed": count,
            "message": f"Compressed {count} file(s)" if count > 0 else "No files to compress",
        }
    except CompressionAlreadyRunning:
        logger.info("Manual compression skipped: compression already running")
        return {
            "success": False,
            "compressed": 0,
            "message": "Compression already running",
        }
    except Exception as e:
        logger.warning(f"Manual compression error: {e}")
        return {
            "success": False,
            "compressed": 0,
            "message": f"Compression failed: {str(e)}",
        }


def delete_log_files(filenames):
    """
    Delete specified log files with security validation.
    Returns dict with result info.
    """
    if not filenames or not isinstance(filenames, list):
        return {"success": False, "deleted": 0, "message": "No files specified"}

    deleted = 0
    errors = []

    for name in filenames:
        filepath, err = _resolve_log_path(name)
        if err is _EXT_DENIED:
            errors.append(f"Unsupported extension: {name}")
            continue
        if err is not None:
            errors.append(err)
            continue

        try:
            os.remove(filepath)
            deleted += 1
            logger.info(f"Deleted log file: {name}")
        except OSError as e:
            errors.append(f"Failed to delete {name}: {e}")

    message = f"Deleted {deleted} file(s)"
    if errors:
        message += f"; errors: {'; '.join(errors)}"

    return {
        "success": deleted > 0 or len(errors) == 0,
        "deleted": deleted,
        "message": message,
    }


def start_auto_compress_daemon(db):
    """Start a daemon thread that runs auto-compression and auto-cleanup periodically."""
    def _worker():
        while True:
            try:
                run_auto_compression(db)
                run_auto_cleanup(db)
            except Exception as e:
                logger.warning(f"Auto compress/cleanup daemon error: {e}")
            time.sleep(COMPRESS_CHECK_INTERVAL)

    thread = threading.Thread(target=_worker, daemon=True, name="log-auto-compress")
    thread.start()
    logger.info("Auto-compress/cleanup daemon started (interval: 5min)")
    return thread