"""Stage 2: Filter — Deterministic noise removal before LLM.""" def filter_repos(repos, config): """ Apply deterministic rules to filter out irrelevant repos. Goal: eliminate obvious noise, not perfect classification. """ filter_config = config.get("filter", {}) reject_categories = set(filter_config.get("reject", [])) keep_categories = set(filter_config.get("keep", [])) min_stars = filter_config.get("min_stars", 50) max_age_days = filter_config.get("max_age_days", 365) from datetime import datetime, timedelta kept = [] rejected = [] now = datetime.now() for repo in repos: if repo.get("status") == "ERROR": continue reasons = [] # Archived if repo.get("archived", False): reasons.append("archived") # Stars too low if repo.get("stars", 0) < min_stars: reasons.append(f"stars {repo.get('stars', 0)} < {min_stars}") # Too old updated = repo.get("updated_at", "") if updated: try: updated_dt = datetime.fromisoformat(updated.replace("Z", "+00:00")) if (now - updated_dt).days > max_age_days: reasons.append(f"too old ({(now - updated_dt).days} days)") except: pass # Wrong category lang = repo.get("language", "") if lang in reject_categories: reasons.append(f"rejected language: {lang}") # Size check — too small to have meaningful workflow size_kb = repo.get("size_kb", 0) if size_kb < 20: reasons.append(f"too small ({size_kb}KB)") # Check description for obvious non-AI content desc_lower = (repo.get("description") or "").lower() skip_keywords = ["css", "animation library", "color picker", "bootstrap theme", "game", "minecraft", "pygame", "flappy bird", "snake game", "dataset", "kaggle", "csv only", "data dump"] for kw in skip_keywords: if kw in desc_lower: reasons.append(f"description contains: {kw}") break if reasons: rejected.append({ "repo": repo, "decision": "REJECT", "reasons": reasons, }) else: repo["status"] = "FILTERED" kept.append(repo) return { "status": "OK", "kept": kept, "rejected": rejected, "kept_count": len(kept), "rejected_count": len(rejected), }