#!/usr/bin/env python3 """ BC Multi-Municipality Development Permit Agent Monitors 8 Lower Mainland / Okanagan municipal websites for new permit applications. Outputs: Excel spreadsheet, HTML dashboard, and Outlook email alerts. Municipalities covered: 1. Surrey – PDF (DP-IN-PROCESS.pdf) 2. Langley (TOL) – HTML portal (e-connect.tol.ca) 3. Abbotsford – ArcGIS REST API (open data) 4. Chilliwack – HTML page with weekly PDF links 5. Burnaby – PDF (Major-Development-Projects.pdf) 6. Richmond – PDF (2025currentdevelopment75120.pdf) 7. Coquitlam – ArcGIS REST API 8. Delta – CivicWeb HTML portal Run weekly. See README.txt for setup instructions. """ import os, json, re, hashlib, smtplib, io, time, html from datetime import datetime, timedelta from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.base import MIMEBase from email import encoders import urllib.request import urllib.parse # ─── CONFIGURATION ──────────────────────────────────────────────────────────── CONFIG = { "email": { "enabled": True, "smtp_server": "smtp.office365.com", "smtp_port": 587, "sender_email": "YOUR_EMAIL@yourdomain.com", # ← Change this "sender_password": "YOUR_APP_PASSWORD", # ← Change this "recipient_emails": ["YOUR_EMAIL@yourdomain.com"], # ← Change this }, # ── Monday.com CRM integration ───────────────────────────────────────────── # Set enabled=True after filling in api_token and board_id. # Get your column IDs from Monday: Board → any column header → Column Settings → Column ID "monday": { "enabled": False, "api_token": "", # ← Your Monday.com API token (v2 Bearer) "board_id": "", # ← Target board ID (integer, shown in board URL) "group_id": "new_group", # ← Group on the board (default: first group) "auto_score_threshold": 60, # ← Auto-push permits scoring above this "column_map": { # Monday column ID → permit field name # These column IDs are examples — replace with your actual board's IDs "text0": "municipality", "text1": "app_number", "text2": "permit_type", "long_text0": "description", "date4": "date_found", "link": "source_url", "numbers0": "score", }, }, # ── Slack webhook alerts ─────────────────────────────────────────────────── # Create an Incoming Webhook at api.slack.com/apps → Your App → Incoming Webhooks "slack": { "enabled": False, "webhook_url": "", # ← Slack Incoming Webhook URL "score_threshold": 60, # ← Only alert for permits scoring above this "permit_types": [], # ← [] = all types; or e.g. ["Mixed Use","Industrial"] }, # ── Team members (shown in pipeline assignee dropdown) ───────────────────── "team_members": [], # ← e.g. ["Sarah", "James", "Mike"] "data_dir": os.path.dirname(os.path.abspath(__file__)), } DATA_DIR = CONFIG["data_dir"] DB_FILE = os.path.join(DATA_DIR, "permits_database.json") EXCEL_FILE = os.path.join(DATA_DIR, "permits_export.xlsx") DASHBOARD_FILE = os.path.join(DATA_DIR, "dashboard.html") LOG_FILE = os.path.join(DATA_DIR, "agent_log.txt") HASH_DIR = os.path.join(DATA_DIR, "hashes") os.makedirs(HASH_DIR, exist_ok=True) # ─── FEATURE FLAGS ──────────────────────────────────────────────────────────── GENERATE_MAP = True # Set to True to include the interactive map tab in the dashboard # (embeds Leaflet JS — makes the HTML file much larger/slower to open) # ─── MUNICIPALITY SOURCES ───────────────────────────────────────────────────── MUNICIPALITIES = [ { "id": "surrey", "name": "City of Surrey", "type": "pdf", "url": "https://www.surrey.ca/sites/default/files/media/documents/DP-IN-PROCESS.pdf", "color": "#003366", }, { "id": "langley", "name": "Township of Langley", "type": "html_portal", "url": "https://services5.arcgis.com/frpHL0Fv8koQRVWY/arcgis/rest/services/Development_Activity_Status_Table/FeatureServer/1/query?where=1%3D1&outFields=Project_Number,Folder_Number,Location,Application_Date,Project_Description,Folder_Status,Community,OurCityLink&f=json&resultRecordCount=2000", "color": "#8B0000", }, { "id": "abbotsford", "name": "City of Abbotsford", "type": "arcgis", "url": "https://maps.abbotsford.ca/arcgis/rest/services/GeocortexExt/WebMap/MapServer/48/query", "color": "#005A9C", }, { "id": "chilliwack", "name": "City of Chilliwack", "type": "html_weekly", "url": "https://www.chilliwack.com/main/page.cfm?id=2186", "color": "#006400", }, { "id": "burnaby", "name": "City of Burnaby", "type": "pdf", "url": "https://www.burnaby.ca/sites/default/files/acquiadam/2024-08/Major-Development-Projects.pdf", "color": "#4B0082", }, { "id": "richmond", "name": "City of Richmond", "type": "pdf", "url": "https://www.richmond.ca/__shared/assets/2025currentdevelopment75120.pdf", "color": "#8B4513", }, { "id": "coquitlam", "name": "City of Coquitlam", "type": "arcgis_coq", "url": "https://services2.arcgis.com/Q6Lq3evZUGfPrN7o/arcgis/rest/services/Development_Information_Demo/FeatureServer/0", "color": "#2E4057", }, { "id": "delta", "name": "City of Delta", "type": "arcgis_delta", "url": "https://mw1.delta.ca/arcgis/rest/services/DeltaMap/Permits/FeatureServer/0", "color": "#8B6914", }, { "id": "calgary", "name": "City of Calgary", "type": "soda", "url": "https://data.calgary.ca/resource/6933-unw5.json", "color": "#CC0000", }, { "id": "victoria", "name": "City of Victoria", "type": "arcgis_victoria", "url": "https://maps.victoria.ca/server/rest/services/OpenData/OpenData_PlanningAndDevelopment/MapServer/3", "color": "#0E7490", }, { "id": "nanaimo", "name": "City of Nanaimo", "type": "arcgis_nanaimo", "url": "https://nanmap.nanaimo.ca/arcgis/rest/services/NanMap/Points/MapServer", "color": "#7C3AED", }, { "id": "squamish", "name": "District of Squamish", "type": "arcgis_squamish", "url": "https://maps.squamish.ca/arcgis/rest/services/Maps/DS_Active_Applications/MapServer", "color": "#059669", }, { "id": "kamloops", "name": "City of Kamloops", "type": "arcgis_kamloops", "url": "https://maps.kamloops.ca/arcgis/rest/services/FeatureDataset/GIS_Development_1/MapServer/87", "color": "#B45309", }, { "id": "langley_city", "name": "City of Langley", "type": "html_langley_city", "url": "https://www.langleycity.ca/business-development/development/development-application-portal", "color": "#E11D48", }, { "id": "langford", "name": "City of Langford", "type": "escribemeetings_langford", "url": "https://pub-langford.escribemeetings.com/feed.rss", "color": "#0369A1", }, # Kelowna: entire kelowna.ca domain behind Cloudflare WAF (HTTP 403 for all bots). # No public API, no eScribe/Granicus/CivicWeb portal found. Disabled until a # scrape-friendly source is identified (e.g. headless browser or open data API). # { # "id": "kelowna", # "name": "City of Kelowna", # "type": "escribemeetings_kelowna", # "url": "https://pub-kelowna.escribemeetings.com/feed.rss", # "color": "#D97706", # }, ] # ─── LOGGING ────────────────────────────────────────────────────────────────── def log(msg): ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S") line = f"[{ts}] {msg}" print(line) with open(LOG_FILE, "a", encoding="utf-8") as f: f.write(line + "\n") # ─── HTTP HELPERS ───────────────────────────────────────────────────────────── HEADERS = { "User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/122.0.0.0 Safari/537.36"), "Accept": "*/*", "Accept-Language": "en-CA,en;q=0.9", } def http_get(url, as_bytes=False, timeout=30, extra_headers=None): try: headers = {**HEADERS, **(extra_headers or {})} req = urllib.request.Request(url, headers=headers) with urllib.request.urlopen(req, timeout=timeout) as r: data = r.read() return data if as_bytes else data.decode("utf-8", errors="replace") except Exception as e: log(f" HTTP GET failed for {url[:80]}: {e}") return None def make_id(*parts): return hashlib.md5("||".join(str(p) for p in parts).encode()).hexdigest()[:14] def get_hash(data): if isinstance(data, str): data = data.encode() return hashlib.md5(data).hexdigest() def load_hash(muni_id): path = os.path.join(HASH_DIR, f"{muni_id}.txt") return open(path).read().strip() if os.path.exists(path) else None def save_hash(muni_id, h): with open(os.path.join(HASH_DIR, f"{muni_id}.txt"), "w") as f: f.write(h) # ─── DATE HELPER ───────────────────────────────────────────────────────────── def _parse_date_str(s): for fmt in ("%d-%b-%y", "%d-%b-%Y"): try: return datetime.strptime(s, fmt).strftime("%Y-%m-%d") except: pass return s # ─── PERMIT TYPE INFERENCE ──────────────────────────────────────────────────── # Types are inferred from description text via keyword matching (priority order). # Burnaby already has exact types from its PDF; those are preserved as-is. TYPE_KEYWORDS = [ ("Mixed Use", ["mixed use", "mixed-use"]), ("Industrial", ["industrial", "warehouse", "warehousing", "manufacturing", "logistics", "distribution", "self-storage"]), ("Multifamily", ["multi-residential", "multifamily", "multi-family", "apartment", "condo", "condominium", "townhouse", "townhome", "duplex", "strata", "multi-unit", "multiplex", "high-rise", "mid-rise", "rental housing", "residential units", "residential building"]), ("Commercial", ["commercial", "retail", "restaurant", "hotel", "motel", "childcare", "daycare", "car wash", "gas station", "grocery", "service station"]), ("Office", ["office"]), ("Institutional", ["school", "church", "institutional", "community centre", "community center", "civic centre", "library", "hospital", "medical", "clinic", "place of worship"]), ("Subdivision", ["subdivision", "rezoning", "rezone", "land use", "lot subdivision", "parcel subdivision"]), ("Residential", ["residential", "single family", "single-family", "detached", "dwelling"]), ] STANDARD_TYPES = {"Mixed Use", "Industrial", "Multifamily", "Commercial", "Office", "Institutional", "Subdivision", "Residential"} def infer_permit_type(description, existing_type=""): """ Return a standardised permit type. - If existing_type is already one of our standard names, keep it. - If existing_type is a non-empty verbose label (e.g. Burnaby's "Multi-Family Residential"), normalise it via keyword matching. - Otherwise infer from description text. """ et = (existing_type or "").strip() # Already a standard type — nothing to do if et in STANDARD_TYPES: return et # Non-empty but non-standard (e.g. Burnaby verbose label) — normalise it if et: et_lower = et.lower() for type_name, keywords in TYPE_KEYWORDS: if any(kw in et_lower for kw in keywords): return type_name # Infer from description text = (description or "").lower() for type_name, keywords in TYPE_KEYWORDS: if any(kw in text for kw in keywords): return type_name return "" # ─── LEAD SCORING ───────────────────────────────────────────────────────────── UNIT_RE = re.compile(r'(\d[\d,]*)\s*(?:residential\s+)?units?', re.IGNORECASE) def score_permit(p): """Return an integer 0-100 sales-lead score. Re-run every scraper pass so the recency bonus decays naturally without any special handling.""" score = 0 ptype = (p.get("permit_type") or "").strip() desc = (p.get("description") or "").lower() # Type weight type_scores = { "Mixed Use": 35, "Industrial": 30, "Commercial": 18, "Office": 15, "Multifamily": 20, "Subdivision": 10, "Institutional": 5, "Residential": 3, } score += type_scores.get(ptype, 0) # Unit count bonus (extracted from description) unit_counts = [int(m.replace(",", "")) for m in UNIT_RE.findall(desc) if m] max_units = max(unit_counts) if unit_counts else 0 if ptype == "Multifamily" and max_units < 50: score -= 8 # small multifamily less interesting if max_units >= 200: score += 25 elif max_units >= 100: score += 18 elif max_units >= 50: score += 10 elif max_units >= 10: score += 5 elif unit_counts: score += 2 # Recency bonus df = p.get("date_found", "") if df: try: age = (datetime.now() - datetime.strptime(df, "%Y-%m-%d")).days if age <= 3: score += 15 elif age <= 7: score += 10 elif age <= 14: score += 5 except Exception: pass # Municipality bonus (Orion's core markets) muni = p.get("municipality", "") if muni in ("City of Surrey", "City of Burnaby", "City of Coquitlam", "City of Richmond"): score += 5 elif muni in ("City of Calgary", "City of Delta", "Township of Langley"): score += 3 return min(100, max(0, score)) # ─── SURREY PLR URL HELPER ──────────────────────────────────────────────────── def make_surrey_plr_url(app_num): """ Convert a Surrey app number like '25 000383' into the direct planning report PDF URL: PLR_7925-0383-00.pdf Pattern: '79' + 2-digit-year + '-' + 4-digit-permit-num + '-00' """ clean = re.sub(r'[\s\-]', '', str(app_num)) if len(clean) >= 7: year = clean[:2] # e.g. "25" num = clean[2:] # e.g. "000383" stripped = num.lstrip('0') or '0' num4 = stripped.zfill(4) # e.g. "0383" return (f"https://www.surrey.ca/sites/default/files/" f"planning-reports/PLR_79{year}-{num4}-00.pdf") return "" # ─── PARSER 1: PDF municipalities (Surrey, Burnaby, Richmond) ───────────────── SURREY_ROW_RE = re.compile( r'^(\d{2}\s+\d{6})\s+' r'(\d{2}-\w{3}-\d{2,4})\s+' r'(\d+\S*\s+[\w\s\.]+?(?:Ave|Avenue|Street|St|Blvd|Boulevard|Road|Rd|Drive|Dr|Way|' r'Crescent|Cres|Place|Pl|Lane|Highway|Hwy|Parkway|Pkwy)[\w\s\.]*?)\s+' r'(.+)$', re.IGNORECASE ) def parse_pdf_permits(pdf_bytes, muni_name): try: import pdfplumber except ImportError: os.system("pip install pdfplumber --break-system-packages -q") import pdfplumber permits = [] try: with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf: full_text = "" for page in pdf.pages: t = page.extract_text() if t: full_text += t + "\n" lines = [l.strip() for l in full_text.split("\n") if l.strip()] if "Surrey" in muni_name: i = 0 while i < len(lines): line = lines[i] m = SURREY_ROW_RE.match(line) if m: app_num = m.group(1).strip() date_sub = _parse_date_str(m.group(2).strip()) address = m.group(3).strip() desc = m.group(4).strip() j = i + 1 while j < len(lines) and not SURREY_ROW_RE.match(lines[j]): next_line = lines[j] if re.match(r'^\d{2}\s+\d{6}', next_line): break if len(next_line) > 5: desc += " " + next_line j += 1 desc_clean = desc.strip() permits.append({ "municipality": muni_name, "app_number": app_num, "date_submitted": date_sub, "address": address, "description": desc_clean, "permit_type": infer_permit_type(desc_clean), "status": "In Process", "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": "", # set by scrape_municipality to the PDF URL "id": make_id(muni_name, app_num, address), }) i = j continue i += 1 log(f" Surrey PDF: parsed {len(permits)} permits") return permits addr_re = re.compile( r"^\d[\w\s\-\.]*(?:Ave|Avenue|St|Street|Blvd|Boulevard|Rd|Road|Dr|Drive|Way|Cres|Crescent|Pl|Place|Lane|Hwy|Highway|Pkwy|Parkway)", re.IGNORECASE ) app_re = re.compile(r"^\d{2,6}[-–]\d{3,6}", re.IGNORECASE) i = 0 while i < len(lines): line = lines[i] if addr_re.match(line) or app_re.match(line): address = "" app_number = "" description = "" if app_re.match(line): app_number = line.split()[0] if line.split() else line address = " ".join(line.split()[1:]) if len(line.split()) > 1 else "" else: address = line lookahead = [] for j in range(1, 4): if i + j < len(lines): lookahead.append(lines[i + j]) for lh in lookahead: if app_re.match(lh) and not app_number: app_number = lh.split()[0] elif not description and len(lh) > 10 and not addr_re.match(lh): description = lh desc_clean = description.strip() permits.append({ "municipality": muni_name, "address": address.strip(), "app_number": app_number.strip(), "date_submitted": "", "description": desc_clean, "permit_type": infer_permit_type(desc_clean), "status": "In Process", "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": "", "id": make_id(muni_name, address, app_number), }) i += max(2, len(lookahead)) else: i += 1 except Exception as e: log(f" PDF parse error for {muni_name}: {e}") return permits # ─── PARSER 1b: Burnaby Major Development Projects PDF ─────────────────────── # Columns (matching your example rows exactly): # Municipality | Plan Areas | Type | App Number | Developer/Architect | # Address | Description | Residential Strata Units | Residential Rental Units | # Status | Date Found | Source URL # # The Burnaby PDF is a structured table. We use pdfplumber table extraction # to respect column boundaries rather than reading raw text which scrambles columns. # Table columns in PDF: Rezoning | Developer/Architect | Address | Development | Units | Status BURNABY_APP_RE = re.compile(r'\bREZ\s*#?\s*(\d{2}[-–]\d{2,4})\b', re.IGNORECASE) BURNABY_PLAN_AREAS = [ "Brentwood", "Metrotown", "Lougheed", "Edmonds", "Burnaby Lake", "Deer Lake", "Burnaby Mountain", "Big Bend", "Newcombe", "Willingdon", "Kingsway", "Still Creek", "Central Park", "Bonsor", "Royal Oak", "South Slope", "Cascade", "Forest Grove", "Sixth Street", "Willingdon Lands", ] BURNABY_TYPES = [ "Multiple-Family Residential", "Multi-Family Residential", "Multiple Family Residential", "Single Family Residential", "Industrial", "Commercial", "Institutional", "Mixed-Use", "Mixed Use", "Office", "Rental Residential", "Mixed-Used Commercial", "Multi-Family rental residential", "Ground-Oriented Multi-Family", ] BURNABY_TYPE_RE = re.compile( r'(' + '|'.join(re.escape(t) for t in BURNABY_TYPES) + r')', re.IGNORECASE ) BURNABY_STATUS_RE = re.compile( r'\b(In Process|Third Reading|Fourth Reading|Second Reading|First Reading|' r'Approved|Issued|Completed|Initial Report|Public Hearing|Public Information Meeting|' r'Referred Back|Withdrawn|Active|Final Adoption)\b', re.IGNORECASE ) BURNABY_DATE_RE = re.compile(r'\b(\d{4}\.\d{2}\.\d{2})\b') STRATA_RE = re.compile(r'(\d[\d,]*)\s*(?:[-–T]\d+\s*)?(?:Mkt\s+)?Strata', re.IGNORECASE) RENTAL_RE = re.compile(r'(\d[\d,]*)\s*(?:[-–T]\d+\s*)?(?:Mkt\s+|Non-Mkt\s+)?Rental', re.IGNORECASE) TOTAL_UNITS_RE = re.compile(r'(\d[\d,]*)\s*(?:units?|suites?|dwellings?)', re.IGNORECASE) def _clean(val): """Strip None and whitespace from a cell value.""" if val is None: return "" return str(val).strip() def _extract_burnaby_units(text): strata, rental = "", "" s_m = STRATA_RE.search(text) if s_m: strata = s_m.group(1).replace(",", "") r_m = RENTAL_RE.search(text) if r_m: rental = r_m.group(1).replace(",", "") if not strata and not rental: t_m = TOTAL_UNITS_RE.search(text) if t_m: rental = t_m.group(1).replace(",", "") return strata, rental def parse_burnaby_pdf(pdf_bytes, source_url): """ Parse the Burnaby Major Development Projects PDF. Column x-boundaries measured directly from the PDF: Plan Areas: x0 51 – 180 Type: x0 181 – 262 Rezoning: x0 263 – 336 Developer: x0 337 – 470 Address: x0 471 – 582 Description: x0 583 – 820 Strata Units: x0 821 – 913 Rental Units: x0 914 – 983 Status: x0 984 – 9999 """ try: import pdfplumber except ImportError: os.system("pip install pdfplumber --break-system-packages -q") import pdfplumber # Column boundaries as (start_x, end_x, name) COLS = [ (51, 180, "plan_area"), (181, 260, "permit_type"), (261, 336, "rezoning"), # REZ starts at x0=262.9 (337, 470, "developer"), (471, 582, "address"), (583, 820, "description"), (821, 913, "strata_units"), (914, 983, "rental_units"), (984, 9999, "status"), ] def col_for_x(x): for start, end, name in COLS: if start <= x < end: return name return None permits = [] try: with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf: for page in pdf.pages: words = page.extract_words(x_tolerance=3, y_tolerance=3) if not words: continue # Group words by row (snap top to nearest 2pt) rows = {} for w in words: y = round(w["top"]) # 1pt grid — precise enough for this PDF rows.setdefault(y, []).append(w) # Build row objects: dict of col_name -> text row_list = [] for y in sorted(rows.keys()): row_words = sorted(rows[y], key=lambda w: w["x0"]) row = {} for w in row_words: col = col_for_x(w["x0"]) if col: row[col] = (row.get(col, "") + " " + w["text"]).strip() row["_y"] = y row_list.append(row) # Now merge multi-line rows into permit records. # A new record starts when we see a "rezoning" value (REZ #xx-xx). # We accumulate subsequent lines until the next REZ# appears. rez_re = re.compile(r'(?:REZ\s*)?#\s*(\d[\w\-]+)', re.IGNORECASE) date_re = re.compile(r'\b(\d{4}\.\d{2}\.\d{2})\b') current = None records = [] for row in row_list: rez_val = row.get("rezoning", "") if rez_re.search(rez_val): # Save previous record if current: records.append(current) current = {k: v for k, v in row.items() if k != "_y"} elif current is not None: # Append continuation lines to existing fields for col in ["plan_area", "permit_type", "developer", "address", "description", "strata_units", "rental_units", "status"]: if row.get(col): current[col] = (current.get(col, "") + " " + row[col]).strip() if current: records.append(current) # Convert records to permit dicts for rec in records: rez_raw = rec.get("rezoning", "") rez_re2 = re.compile(r'(?:REZ\s*)?#\s*(\d[\w\-]+)', re.IGNORECASE) m = rez_re2.search(rez_raw) if not m: continue app_num = "REZ#" + m.group(1).replace("–", "-").strip() # Clean status — strip trailing date status_raw = rec.get("status", "In Process") date_str = "" dm = date_re.search(status_raw) if dm: date_str = dm.group(1).replace(".", "-") # Convert YYYY-MM-DD format try: date_str = datetime.strptime(dm.group(1), "%Y.%m.%d").strftime("%Y-%m-%d") except: pass status_clean = status_raw[:status_raw.find(dm.group(1))].strip() else: status_clean = status_raw # Strip trailing date artifacts like "2024.06.24" from status status_clean = re.sub(r'\s*\d{4}\.\d{2}\.\d{2}.*$', '', status_clean).strip() address = rec.get("address", "").strip() permits.append({ "municipality": "City of Burnaby", "plan_area": rec.get("plan_area", "").strip(), "permit_type": rec.get("permit_type", "").strip(), "app_number": app_num, "developer": rec.get("developer", "").strip()[:150], "address": address[:250], "description": rec.get("description", "").strip()[:500], "strata_units": rec.get("strata_units", "").strip()[:200], "rental_units": rec.get("rental_units", "").strip()[:200], "status": status_clean[:80], "date_submitted": date_str, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": source_url, "id": make_id("burnaby", app_num, address), }) except Exception as e: log(f" Burnaby PDF parse error: {e}") # Deduplicate seen = set() unique = [] for p in permits: if p["id"] not in seen: seen.add(p["id"]) unique.append(p) log(f" Burnaby PDF: parsed {len(unique)} permits") return unique # ─── PARSER 2: Langley TOL — ArcGIS Development Activity Status Table ───────── def scrape_langley(): """ Uses the Township of Langley's ArcGIS FeatureServer table (Development Activity Status Table). Returns one record per active application, including a direct link to the application details page via OurCityLink. """ base_url = ("https://services5.arcgis.com/frpHL0Fv8koQRVWY/arcgis/rest/services" "/Development_Activity_Status_Table/FeatureServer/1") params = urllib.parse.urlencode({ "where": "1=1", "outFields": "Project_Number,Folder_Number,Location,Application_Date," "Project_Description,Folder_Status,Community,OurCityLink", "f": "json", "resultRecordCount": 2000, "orderByFields": "OBJECTID DESC", }) raw = http_get(f"{base_url}/query?{params}") if not raw: return [] permits = [] try: data = json.loads(raw) features = data.get("features", []) for feat in features: attrs = feat.get("attributes", {}) location = str(attrs.get("Location") or "").strip() folder_num = str(attrs.get("Folder_Number") or attrs.get("Project_Number") or "").strip() description = str(attrs.get("Project_Description") or "").strip() status = str(attrs.get("Folder_Status") or "Active").strip() community = str(attrs.get("Community") or "").strip() our_city = str(attrs.get("OurCityLink") or "").strip() # Application_Date may come back as unix ms (int) or a date string app_date_raw = attrs.get("Application_Date") app_date = "" if isinstance(app_date_raw, (int, float)) and app_date_raw: try: app_date = datetime.fromtimestamp(app_date_raw / 1000).strftime("%Y-%m-%d") except Exception: pass elif isinstance(app_date_raw, str): app_date = app_date_raw[:10] desc_full = description if community: desc_full = f"{description} | Community: {community}".strip(" |") permits.append({ "municipality": "Township of Langley", "address": location[:200], "app_number": folder_num, "description": desc_full[:400], "permit_type": infer_permit_type(desc_full), "status": status, "date_submitted": app_date, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": our_city or base_url, "id": make_id("langley", folder_num, location), }) log(f" Langley: extracted {len(permits)} permits from FeatureServer") except Exception as e: log(f" Langley ArcGIS parse error: {e}") return permits # ─── PARSER 3: ArcGIS REST API (Abbotsford & Coquitlam) ─────────────────────── def scrape_arcgis(muni_id, muni_name, base_url): params = urllib.parse.urlencode({ "where": "1=1", "outFields": "*", "f": "json", "resultRecordCount": 1000, }) url = f"{base_url}?{params}" raw = http_get(url) if not raw: return [] permits = [] try: data = json.loads(raw) features = data.get("features", []) for feat in features: attrs = feat.get("attributes", {}) address = (attrs.get("ADDRESS") or attrs.get("CIVIC_ADDRESS") or attrs.get("address") or attrs.get("SiteAddress") or attrs.get("LOCATION") or attrs.get("Location") or "") app_num = (attrs.get("FILE_NUMBER") or attrs.get("APPLICATION_NUMBER") or attrs.get("AppNo") or attrs.get("PERMIT_NUMBER") or attrs.get("FOLDER_NUMBER") or attrs.get("DEV_FILE") or str(attrs.get("OBJECTID", ""))) desc = (attrs.get("PROPOSAL") or attrs.get("DESCRIPTION") or attrs.get("Description") or attrs.get("TYPE") or "") status = (attrs.get("STATUS") or attrs.get("STATUS_DESCRIPTION") or attrs.get("Status") or attrs.get("AMANDA_STATUS") or "Active") # Year field present in Abbotsford (no dedicated date field) year_val = attrs.get("YEAR") date_sub = str(year_val) if year_val else "" desc_clean = str(desc).strip()[:300] permits.append({ "municipality": muni_name, "address": str(address).strip()[:200], "app_number": str(app_num).strip(), "description": desc_clean, "permit_type": infer_permit_type(desc_clean), "status": str(status).strip(), "date_submitted": date_sub, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": base_url, "id": make_id(muni_id, address, app_num), }) log(f" {muni_name}: {len(permits)} records from ArcGIS API") except Exception as e: log(f" ArcGIS parse error for {muni_name}: {e}") return permits # ─── PARSER 3b: Coquitlam — ArcGIS FeatureServer ───────────────────────────── def scrape_coquitlam(base_url): """ City of Coquitlam Development Information FeatureServer. Fields: PROJECT_NUMBER, PROJECT_DESCRIPTION, PROJECT_STATUS, APPLICANT, SUBMISSION_DATE (unix ms), ADDRESS """ params = urllib.parse.urlencode({ "where": "1=1", "outFields": "PROJECT_NUMBER,PROJECT_DESCRIPTION,PROJECT_STATUS,APPLICANT,SUBMISSION_DATE,ADDRESS", "f": "json", "resultRecordCount": 2000, "orderByFields": "OBJECTID DESC", }) raw = http_get(f"{base_url}/query?{params}") if not raw: return [] permits = [] try: data = json.loads(raw) features = data.get("features", []) for feat in features: attrs = feat.get("attributes", {}) address = str(attrs.get("ADDRESS") or "").strip() app_num = str(attrs.get("PROJECT_NUMBER") or "").strip() desc = str(attrs.get("PROJECT_DESCRIPTION") or "").strip() status = str(attrs.get("PROJECT_STATUS") or "Active").strip() applicant = str(attrs.get("APPLICANT") or "").strip() sub_raw = attrs.get("SUBMISSION_DATE") date_sub = "" if isinstance(sub_raw, (int, float)) and sub_raw: try: date_sub = datetime.fromtimestamp(sub_raw / 1000).strftime("%Y-%m-%d") except Exception: pass desc_clean = desc[:400] permits.append({ "municipality": "City of Coquitlam", "address": address[:200], "app_number": app_num, "description": desc_clean, "permit_type": infer_permit_type(desc_clean), "status": status, "applicant": applicant, "date_submitted": date_sub, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": base_url, "id": make_id("coquitlam", app_num, address), }) log(f" Coquitlam: extracted {len(permits)} permits from FeatureServer") except Exception as e: log(f" Coquitlam ArcGIS parse error: {e}") return permits # ─── CHILLIWACK VALUE FILTERS ───────────────────────────────────────────────── # Only keep permits matching these criteria: # Multifamily > $5,000,000 # Industrial > $3,000,000 # Commercial > $3,000,000 # Renovation > $300,000 CHILLIWACK_FILTERS = [ {"keywords": ["multifamily", "multi-family", "multi family", "apartment", "residential - multi", "strata", "condo", "townhouse", "townhome", "row house", "duplex", "triplex", "fourplex", "multi-unit", "multiunit"], "min_value": 5_000_000}, {"keywords": ["industrial", "warehouse", "manufacturing", "distribution", "storage", "light industrial", "heavy industrial"], "min_value": 3_000_000}, {"keywords": ["commercial", "retail", "office", "mixed use", "mixed-use", "hotel", "motel", "restaurant", "institutional", "assembly", "place of worship", "school", "care facility", "medical"], "min_value": 3_000_000}, {"keywords": ["renovation", "reno", "alteration", "addition", "interior alteration", "tenant improvement", "ti ", "fit-up", "fitup", "retrofit"], "min_value": 300_000}, ] def _parse_dollar_value(text): """Extract a dollar value from a string like '$1,234,567' or '1234567'.""" text = text.replace(",", "").replace(" ", "") m = re.search(r'\$?([\d]+(?:\.\d+)?)', text) if m: try: return float(m.group(1)) except: pass return 0.0 def _chilliwack_passes_filter(description, value_str): """Return True if this permit meets any of our value/type thresholds.""" value = _parse_dollar_value(value_str) desc_lower = (description or "").lower() for f in CHILLIWACK_FILTERS: if any(kw in desc_lower for kw in f["keywords"]): if value >= f["min_value"]: return True return False # ─── PARSER 4: Chilliwack weekly PDFs (OCR-based, full backfill) ────────────── def _ensure_ocr_deps(): """Install OCR dependencies if not present.""" try: import pytesseract from pdf2image import convert_from_bytes return True except ImportError: log(" Installing OCR dependencies (one-time, may take a minute)...") os.system("pip install pytesseract pdf2image pillow --break-system-packages -q") try: import pytesseract from pdf2image import convert_from_bytes return True except ImportError: log(" OCR install failed. Make sure tesseract is installed: brew install tesseract") return False def _ocr_pdf(pdf_bytes): """Convert a scanned PDF to text using OCR. Returns full text string.""" from pdf2image import convert_from_bytes import pytesseract images = convert_from_bytes(pdf_bytes, dpi=200) full_text = "" for img in images: full_text += pytesseract.image_to_string(img) + "\n" return full_text def _parse_chilliwack_ocr_text(text, source_url, report_label=""): """ Parse OCR text from a Chilliwack issued-permits PDF. Columns are typically: Permit No | Address | Description | Value | Issued Date Applies value/type filters before returning. """ permits = [] lines = [l.strip() for l in text.split("\n") if l.strip()] # Permit numbers look like: BP-2026-00123 or BP2026-00123 or 26-00123 permit_re = re.compile(r'^((?:BP|BLD|B)[-\s]?\d{2,4}[-\s]\d{3,6}|\d{2,4}[-\s]\d{4,6})\b', re.IGNORECASE) # Dollar values value_re = re.compile(r'\$[\d,]+(?:\.\d+)?|\b\d{1,3}(?:,\d{3})+(?:\.\d+)?') i = 0 while i < len(lines): line = lines[i] m = permit_re.match(line) if m: permit_no = m.group(1).strip() rest = line[m.end():].strip() # Collect next few lines to build address + description + value block = [rest] for j in range(1, 5): if i + j < len(lines) and not permit_re.match(lines[i + j]): block.append(lines[i + j]) else: break block_text = " ".join(block) # Extract value value_matches = value_re.findall(block_text) value_str = value_matches[-1] if value_matches else "" # Try to split address from description # Address usually comes first (has a number at the start) addr_re = re.compile(r'^\d+\s+\S+', re.IGNORECASE) address = "" description = "" for part in block: if not address and addr_re.match(part): address = part elif not description and len(part) > 5 and not value_re.match(part): description = part if not address: address = block[0] if block else "" if not description and len(block) > 1: description = block[1] if block[1] != address else (block[2] if len(block) > 2 else "") # Extract date (look for patterns like 2026-03-10 or Mar 10, 2026) date_str = "" date_m = re.search(r'\d{4}-\d{2}-\d{2}|\d{1,2}[-/]\w{3}[-/]\d{2,4}', block_text) if date_m: date_str = _parse_date_str(date_m.group(0)) # Apply filter full_desc = f"{description} {block_text}" if _chilliwack_passes_filter(full_desc, value_str): chilli_desc = f"{description} | Value: {value_str}".strip(" |")[:300] permits.append({ "municipality": "City of Chilliwack", "app_number": permit_no, "date_submitted": date_str, "address": address[:200], "description": chilli_desc, "permit_type": infer_permit_type(chilli_desc), "status": "Issued", "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": source_url, "id": make_id("chilliwack", permit_no, address), }) i += len(block) else: i += 1 return permits def scrape_chilliwack(): """ Fetches ALL Chilliwack weekly permit PDFs listed on the page (up to 1 year). Uses OCR to read scanned PDFs. Tracks processed report IDs to avoid re-processing. Only saves permits matching the value/type thresholds defined in CHILLIWACK_FILTERS. """ if not _ensure_ocr_deps(): log(" Chilliwack: skipping — OCR dependencies not available") return [] url = "https://www.chilliwack.com/main/page.cfm?id=2186" html = http_get(url) if not html: return [] # Get all attachment links — (path, attachID) attach_links = re.findall( r'href=["\']([^"\']*attachView\.cfm\?attachID=(\d+))["\']', html, re.IGNORECASE ) if not attach_links: log(" Chilliwack: no attachment links found") return [] # Load set of already-processed IDs processed_file = os.path.join(HASH_DIR, "chilliwack_processed.json") if os.path.exists(processed_file): with open(processed_file) as f: processed_ids = set(json.load(f)) else: processed_ids = set() # Find unprocessed reports unprocessed = [(path, aid) for path, aid in attach_links if aid not in processed_ids] if not unprocessed: log(" Chilliwack: all reports already processed") return [] log(f" Chilliwack: {len(unprocessed)} unprocessed report(s) to OCR (this may take a few minutes)...") all_permits = [] for path, attach_id in unprocessed: if not path.startswith("http"): path = "https://www.chilliwack.com" + path pdf_bytes = http_get(path, as_bytes=True) if not pdf_bytes: log(f" Chilliwack: failed to download report ID {attach_id}, skipping") continue try: text = _ocr_pdf(pdf_bytes) permits = _parse_chilliwack_ocr_text(text, path, report_label=attach_id) log(f" Chilliwack report {attach_id}: {len(permits)} qualifying permits found") all_permits.extend(permits) except Exception as e: log(f" Chilliwack OCR error on report {attach_id}: {e}") # Mark as processed regardless of result so we don't retry bad PDFs processed_ids.add(attach_id) # Save updated processed list with open(processed_file, "w") as f: json.dump(list(processed_ids), f) log(f" Chilliwack total qualifying permits this run: {len(all_permits)}") return all_permits # ─── PARSER 5: Delta — Monthly Development Highlights PDFs ─────────────────── def scrape_victoria(): """ City of Victoria — Development Applications via ArcGIS MapServer layer 3. Returns point geometry in WGS84 so no geocoding needed. Fields: HOUSE, STREET, FOLDER_NUMBER, AppType, STATUS, PURPOSE, CREATED_DATE, DevAppTracker """ base_url = ("https://maps.victoria.ca/server/rest/services/OpenData" "/OpenData_PlanningAndDevelopment/MapServer/3") params = urllib.parse.urlencode({ "where": "STATUS='ACTIVE'", "outFields": "FOLDER_NUMBER,AppType,HOUSE,STREET,STATUS,PURPOSE,CREATED_DATE,DevAppTracker", "returnGeometry": "true", "outSR": "4326", "f": "json", "resultRecordCount": 2000, }) raw = http_get(f"{base_url}/query?{params}") if not raw: return [] permits = [] try: data = json.loads(raw) features = data.get("features", []) for feat in features: attrs = feat.get("attributes", {}) geom = feat.get("geometry", {}) house = str(attrs.get("HOUSE") or "").strip() street = str(attrs.get("STREET") or "").strip() address = f"{house} {street}".strip() folder = str(attrs.get("FOLDER_NUMBER") or "").strip() app_type = str(attrs.get("AppType") or "").strip() status = str(attrs.get("STATUS") or "Active").strip().title() purpose = str(attrs.get("PURPOSE") or "").strip() created_raw = attrs.get("CREATED_DATE") date_sub = "" if isinstance(created_raw, (int, float)) and created_raw: try: date_sub = datetime.fromtimestamp(created_raw / 1000).strftime("%Y-%m-%d") except Exception: pass desc = f"{app_type} — {purpose}".strip(" —") if purpose else app_type tracker_id = str(attrs.get("DevAppTracker") or "").strip() source = (f"https://www.victoria.ca/building-business/permits-development-construction" f"/development-tracker") p = { "municipality": "City of Victoria", "address": address[:200], "app_number": folder, "description": desc[:400], "permit_type": infer_permit_type(desc), "status": status, "date_submitted": date_sub, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": source, "id": make_id("victoria", folder, address), } # Attach coordinates directly — no geocoding needed if geom.get("x") and geom.get("y"): p["lat"] = geom["y"] p["lng"] = geom["x"] permits.append(p) log(f" Victoria: extracted {len(permits)} active applications from MapServer") except Exception as e: log(f" Victoria ArcGIS parse error: {e}") return permits def scrape_nanaimo(): """ City of Nanaimo — Development Permits (layer 21) + Rezoning Applications (layer 22). ArcGIS MapServer with point geometry in WGS84. Fields: PERMITNUM, APPLICATION_TYPE, SUBJECT (address), DESCRIPTION """ base_url = "https://nanmap.nanaimo.ca/arcgis/rest/services/NanMap/Points/MapServer" permits = [] for layer_id in (21, 22): params = urllib.parse.urlencode({ "where": "1=1", "outFields": "PERMITNUM,APPLICATION_TYPE,SUBJECT,DESCRIPTION", "returnGeometry": "true", "outSR": "4326", "f": "json", "resultRecordCount": 2000, }) raw = http_get(f"{base_url}/{layer_id}/query?{params}") if not raw: continue try: data = json.loads(raw) features = data.get("features", []) for feat in features: attrs = feat.get("attributes", {}) geom = feat.get("geometry", {}) app_num = str(attrs.get("PERMITNUM") or "").strip() app_type = str(attrs.get("APPLICATION_TYPE") or "").strip() address = str(attrs.get("SUBJECT") or "").strip() desc = str(attrs.get("DESCRIPTION") or "").strip() full_desc = f"{app_type} — {desc}".strip(" —") if desc else app_type p = { "municipality": "City of Nanaimo", "address": address[:200], "app_number": app_num, "description": full_desc[:400], "permit_type": infer_permit_type(full_desc), "status": "Active", "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": "https://www.nanaimo.ca/whatsbuilding", "id": make_id("nanaimo", app_num, address), } if geom.get("x") and geom.get("y"): p["lat"] = geom["y"] p["lng"] = geom["x"] permits.append(p) log(f" Nanaimo layer {layer_id}: {len(features)} records") except Exception as e: log(f" Nanaimo layer {layer_id} parse error: {e}") log(f" Nanaimo total: {len(permits)} permits") return permits def scrape_squamish(): """ District of Squamish — Active Development Applications via ArcGIS MapServer. Layer 5: Rezoning (RZ), Layer 6: Subdivision/DP (SD). Fields: Address, DS_Project_Number, Application_Type, Application_Date, Status """ base_url = "https://maps.squamish.ca/arcgis/rest/services/Maps/DS_Active_Applications/MapServer" permits = [] for layer_id, layer_name in ((5, "RZ"), (6, "SD")): params = urllib.parse.urlencode({ "where": "1=1", "outFields": "Address,DS_Project_Number,Application_Type,Application_Sub_Type," "Application_Date,Status,Project_Name", "returnGeometry": "true", "outSR": "4326", "f": "json", "resultRecordCount": 2000, }) raw = http_get(f"{base_url}/{layer_id}/query?{params}") if not raw: log(f" Squamish layer {layer_id} ({layer_name}): no response") continue try: data = json.loads(raw) features = data.get("features", []) for feat in features: attrs = feat.get("attributes", {}) geom = feat.get("geometry", {}) address = str(attrs.get("Address") or "").strip() proj_num = str(attrs.get("DS_Project_Number") or "").strip() app_type = str(attrs.get("Application_Type") or "").strip() sub_type = str(attrs.get("Application_Sub_Type") or "").strip() proj_name = str(attrs.get("Project_Name") or "").strip() status = str(attrs.get("Status") or "Active").strip() date_raw = attrs.get("Application_Date") date_sub = "" if isinstance(date_raw, (int, float)) and date_raw: try: date_sub = datetime.fromtimestamp(date_raw / 1000).strftime("%Y-%m-%d") except Exception: pass elif isinstance(date_raw, str): date_sub = date_raw[:10] desc_parts = [x for x in [app_type, sub_type, proj_name] if x] desc = " — ".join(desc_parts)[:400] p = { "municipality": "District of Squamish", "address": address[:200], "app_number": proj_num, "description": desc, "permit_type": infer_permit_type(desc), "status": status, "date_submitted": date_sub, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": "https://squamish.ca/building-and-land-development/home-land-and-property-development/development-review-and-local-projects/development-review/", "id": make_id("squamish", proj_num, address), } if geom.get("x") and geom.get("y"): p["lat"] = geom["y"] p["lng"] = geom["x"] permits.append(p) log(f" Squamish layer {layer_id} ({layer_name}): {len(features)} records") except Exception as e: log(f" Squamish layer {layer_id} parse error: {e}") log(f" Squamish total: {len(permits)} permits") return permits def scrape_kamloops(): """ City of Kamloops — Active Development projects via ArcGIS MapServer layer 87. Filters to TYPE='DEVELOPMENT - CURRENT/ACTIVE' (68 records). Geometry is polygons; lat/lng derived from ring centroid. """ base_url = "https://maps.kamloops.ca/arcgis/rest/services/FeatureDataset/GIS_Development_1/MapServer/87" params = urllib.parse.urlencode({ "where": "TYPE='DEVELOPMENT - CURRENT/ACTIVE'", "outFields": "OBJECTID,TYPE,DESCRIPTION,CREATED_DATE", "returnGeometry": "true", "outSR": "4326", "f": "json", "resultRecordCount": 1000, }) raw = http_get(f"{base_url}/query?{params}") if not raw: log(" Kamloops: no response") return [] permits = [] try: data = json.loads(raw) features = data.get("features", []) for feat in features: attrs = feat.get("attributes", {}) geom = feat.get("geometry", {}) obj_id = str(attrs.get("OBJECTID", "")).strip() desc = str(attrs.get("DESCRIPTION") or "").strip() date_raw = attrs.get("CREATED_DATE") date_sub = "" if isinstance(date_raw, (int, float)) and date_raw: try: date_sub = datetime.fromtimestamp(date_raw / 1000).strftime("%Y-%m-%d") except Exception: pass # Compute polygon centroid from first ring lat, lng = None, None rings = geom.get("rings", []) if rings: pts = rings[0] if pts: lng = sum(p[0] for p in pts) / len(pts) lat = sum(p[1] for p in pts) / len(pts) p = { "municipality": "City of Kamloops", "address": "Kamloops, BC", "app_number": f"KAM-{obj_id}", "description": desc, "permit_type": infer_permit_type(desc), "status": "Active", "date_submitted": date_sub, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": "https://kamloops.maps.arcgis.com/apps/instant/basic/index.html?appid=2c7e701a77514f1d98d98b6c1484f023", "id": make_id("kamloops", obj_id, desc), } if lat is not None and lng is not None: p["lat"] = lat p["lng"] = lng permits.append(p) log(f" Kamloops: {len(permits)} active development records") except Exception as e: log(f" Kamloops parse error: {e}") return permits def scrape_langley_city(): """ City of Langley — Development Application Portal (Drupal accordion HTML page). URL: https://www.langleycity.ca/business-development/development/development-application-portal Each accordion item: ADDRESS (PERMIT_NUM) → status, description, app type, date submitted. """ url = "https://www.langleycity.ca/business-development/development/development-application-portal" raw = http_get(url) if not raw: log(" City of Langley: no response") return [] permits = [] try: # Split on accordion trigger buttons — each item has heading + body items = re.split(r']+class="[^"]*accordion__trigger[^"]*"', raw) for item in items[1:]: # Heading: aria-label="ADDRESS (PERMIT_NUM), Show this section" heading_match = re.search(r'aria-label="([^"]+), Show this section"', item) if not heading_match: continue # Double-unescape handles &amp; → & → & heading = html.unescape(html.unescape(heading_match.group(1))).strip() # Detect permit number — could be "ADDRESS (PERMIT)" or "PERMIT (ADDRESS)" PERMIT_RE = re.compile(r'\b(?:DP|RZ|OCP|SD|VP)\s*\d{2}[-/]\d{2}', re.IGNORECASE) perm_match = re.search(r'\(([^)]+)\)\s*$', heading) if perm_match: inside_parens = perm_match.group(1).strip() before_parens = heading[:heading.rfind("(")].strip().rstrip(",").strip() # If the parenthetical looks like a permit code, it IS the permit number if PERMIT_RE.search(inside_parens): permit_num = inside_parens address = before_parens elif PERMIT_RE.search(before_parens): # Reversed: permit is before parens, address is inside permit_num = before_parens address = inside_parens else: permit_num = inside_parens address = before_parens else: permit_num = "" address = heading # Current status status = "" s_match = re.search(r'Current status[^:]*:[^>]*>([^<]+)', item) if s_match: status = html.unescape(s_match.group(1)).strip().strip("\u00a0") # Description desc = "" d_match = re.search(r'Description[^:]*:\s*([^<]+)', item) if d_match: desc = html.unescape(d_match.group(1)).strip() # Application type app_type = "" at_match = re.search(r'Application type[^:]*:\s*([^<]+)', item) if at_match: app_type = html.unescape(at_match.group(1)).strip().strip("\u00a0") # Submission date date_sub = "" date_match = re.search(r'Application submitted[^:]*:\s*([^<]+)', item) if date_match: date_raw = html.unescape(date_match.group(1)).strip().strip("\u00a0") try: date_sub = datetime.strptime(date_raw, "%B %d, %Y").strftime("%Y-%m-%d") except Exception: date_sub = date_raw[:20] if not permit_num and not address: continue address_full = f"{address}, Langley, BC" if address and "langley" not in address.lower() else address p = { "municipality": "City of Langley", "address": address_full[:200], "app_number": permit_num, "description": desc[:400], "permit_type": infer_permit_type(f"{app_type} {desc}"), "status": status[:200], "date_submitted": date_sub, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": url, "id": make_id("langley_city", permit_num, address), } permits.append(p) log(f" City of Langley: {len(permits)} permits") except Exception as e: log(f" City of Langley parse error: {e}") return permits def _escribemeetings_fetch(url): """Fetch an eScribe page via urllib with SSL cert verification disabled. eScribe's certificate chain fails Python 3.14 verification on macOS.""" import ssl ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE try: req = urllib.request.Request(url, headers=HEADERS) resp = urllib.request.urlopen(req, timeout=20, context=ctx) return resp.read().decode("utf-8", errors="replace") except Exception as e: log(f" eScribe fetch failed {url[:80]}: {e}") return None def _parse_escribemeetings(city_name, city_id, rss_url, muni_url): """ Generic eScribe council minutes scraper. Reads the RSS feed → for each Council or SDAC meeting with an agenda link, fetches the agenda HTML and extracts planning-related agenda items (Development Permit, Rezoning, OCP Amendment, DVP, Temporary Use Permit). Only processes meetings not yet seen (hash-tracked per meeting ID). """ PERMIT_KEYWORDS = [ "development permit", "rezoning", "ocp amendment", "official community plan amendment", ] MEETING_TYPES = {"council meeting", "special council meeting", "sustainable development advisory committee", "public hearing"} rss_raw = _escribemeetings_fetch(rss_url) if not rss_raw: return [] items = re.findall(r"(.*?)", rss_raw, re.DOTALL) permits = [] for item_xml in items: # Meeting type meeting_tag = re.search(r']+/>', item_xml, re.DOTALL) if not meeting_tag: continue meeting_type_match = re.search(r'Type="([^"]+)"', meeting_tag.group(0)) if not meeting_type_match: continue meeting_type = meeting_type_match.group(1).strip() if meeting_type.lower() not in MEETING_TYPES: continue # Meeting ID and agenda link meeting_id_match = re.search(r'Id="([^"]+)"', meeting_tag.group(0)) if not meeting_id_match: continue meeting_id = meeting_id_match.group(1).strip() link_match = re.search(r"(.*?)", item_xml) link = link_match.group(1).strip() if link_match else "" if "Meeting?Id=" not in link: continue # agenda not yet published # Meeting date date_match = re.search(r'Start="(\d+/\d+/\d+)', meeting_tag.group(0)) meeting_date = "" if date_match: try: meeting_date = datetime.strptime( date_match.group(1), "%m/%d/%Y" ).strftime("%Y-%m-%d") except Exception: meeting_date = date_match.group(1) # Skip if we've already processed this meeting cache_key = f"{city_id}_{meeting_id}" if load_hash(cache_key): continue # Fetch agenda HTML agenda_url = (f"https://pub-{city_id}.escribemeetings.com/" f"Meeting?Id={meeting_id}&Agenda=Agenda&lang=English") agenda_html = _escribemeetings_fetch(agenda_url) if not agenda_html: continue # Save hash so we don't re-process this meeting save_hash(cache_key, get_hash(meeting_id)) # Extract agenda item titles (h2/h3 inside AgendaItemTitle divs) agenda_items = re.findall( r"class='AgendaItemTitle'[^>]*>]+>([^<]+)", agenda_html ) for item_title in agenda_items: item_title = item_title.strip() if not any(kw in item_title.lower() for kw in PERMIT_KEYWORDS): continue # Parse: "Permit Type - Address" or just title parts = item_title.split(" - ", 1) if len(parts) == 2: permit_type_raw, address = parts[0].strip(), parts[1].strip() else: permit_type_raw = item_title address = f"{city_name}, BC" address_full = address if any( x in address.lower() for x in ["langford", "kelowna", "bc", "ave", "st", "rd", "drive", "place", "court", "way"] ) else f"{address}, {city_name}, BC" desc = f"{permit_type_raw} — from {meeting_type} agenda ({meeting_date})" p = { "municipality": city_name, "address": address_full[:200], "app_number": "", "description": desc[:400], "permit_type": infer_permit_type(f"{permit_type_raw} {item_title}"), "status": "Council Agenda", "date_submitted": meeting_date, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": link, "id": make_id(city_id, meeting_id, item_title), } permits.append(p) log(f" {city_name} — meeting {meeting_date} ({meeting_type}): " f"{len([p for p in permits if p['date_submitted'] == meeting_date])} permit items") log(f" {city_name} total: {len(permits)} new permit agenda items") return permits def scrape_delta(): """ City of Delta building permits via ArcGIS FeatureServer. https://mw1.delta.ca/arcgis/rest/services/DeltaMap/Permits/FeatureServer/0 Fields: CIVIC_ADDRESS, PERMITNUMBER, TYPE, SUBJECT, ADDRESS, STATUS, COMPLETED_DATE Minor trade permits (plumbing, electrical, gas, mechanical) are excluded. """ base_url = "https://mw1.delta.ca/arcgis/rest/services/DeltaMap/Permits/FeatureServer/0" # Actual TYPE values from Delta: COMMERCIAL, MULTI-RESIDENTIAL, RESIDENTIAL, # DEMOLITION - OTHER, DEMOLITION - RESIDENTIAL, PLUMBING PERMIT # Exclude plumbing-only permits; keep all structural/development work params = urllib.parse.urlencode({ "where": "TYPE NOT IN ('PLUMBING PERMIT')", "outFields": "PERMITNUMBER,TYPE,SUBJECT,CIVIC_ADDRESS,ADDRESS,STATUS,COMPLETED_DATE", "f": "json", "resultRecordCount": 2000, "orderByFields": "OBJECTID DESC", }) raw = http_get(f"{base_url}/query?{params}") if not raw: log(" Delta: could not reach ArcGIS FeatureServer") return [] permits = [] try: data = json.loads(raw) features = data.get("features", []) for feat in features: attrs = feat.get("attributes", {}) address = str(attrs.get("CIVIC_ADDRESS") or attrs.get("ADDRESS") or "").strip() app_num = str(attrs.get("PERMITNUMBER") or "").strip() ptype = str(attrs.get("TYPE") or "").strip() subject = str(attrs.get("SUBJECT") or "").strip() status = str(attrs.get("STATUS") or "Active").strip() # COMPLETED_DATE is unix ms date_sub = "" comp_raw = attrs.get("COMPLETED_DATE") if isinstance(comp_raw, (int, float)) and comp_raw: try: date_sub = datetime.fromtimestamp(comp_raw / 1000).strftime("%Y-%m-%d") except Exception: pass desc = f"{ptype} — {subject}".strip(" —") if subject else ptype permits.append({ "municipality": "City of Delta", "address": address[:200], "app_number": app_num, "description": desc[:400], "permit_type": infer_permit_type(desc), "status": status, "date_submitted": date_sub, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": "https://www.delta.ca/building-development", "id": make_id("delta", app_num, address), }) log(f" Delta: extracted {len(permits)} permits from ArcGIS FeatureServer") except Exception as e: log(f" Delta ArcGIS parse error: {e}") return permits # ─── PARSER 6: Calgary — Socrata Open Data API (SODA) ──────────────────────── # Calgary's open data portal has 185k+ development permits going back to 2006. # We filter to commercial / industrial / multi-residential / mixed-use types # submitted in the last 2 years to keep the dataset focused on leads. CALGARY_INCLUDE_KEYWORDS = [ "multi-residential", "multi residential", "commercial", "industrial", "mixed use", "mixed-use", "office", "institutional", "hotel", "retail", "warehouse", "manufacturing", "high density", "medium density", ] def scrape_calgary(soda_url): """ Fetches development permits from Calgary's Socrata (SODA) open data API. Filters to commercial/industrial/multi-residential types from the last 2 years. Calgary already provides lat/lng in the dataset — no geocoding needed. """ cutoff = (datetime.now().replace(year=datetime.now().year - 2)).strftime("%Y-%m-%dT00:00:00.000") where = (f"applieddate > '{cutoff}' AND " f"(statuscurrent != 'Cancelled' AND statuscurrent != 'Withdrawn')") params = urllib.parse.urlencode({ "$where": where, "$select": "permitnum,applieddate,statuscurrent,address,communityname," "category,description,applicant,latitude,longitude", "$order": "applieddate DESC", "$limit": 5000, }) raw = http_get(f"{soda_url}?{params}") if not raw: return [] permits = [] try: records = json.loads(raw) for r in records: category = (r.get("category") or "").lower() # Filter to relevant project types only if not any(kw in category for kw in CALGARY_INCLUDE_KEYWORDS): continue app_date_raw = r.get("applieddate", "") app_date = app_date_raw[:10] if app_date_raw else "" # Lat/lng already in the dataset lat, lng = None, None try: lat = float(r["latitude"]) lng = float(r["longitude"]) except (KeyError, TypeError, ValueError): pass address = str(r.get("address") or "").strip() community = str(r.get("communityname") or "").strip() app_num = str(r.get("permitnum") or "").strip() desc = str(r.get("description") or "").strip() applicant = str(r.get("applicant") or "").strip() status = str(r.get("statuscurrent") or "Active").strip() cat_disp = str(r.get("category") or "").strip() full_address = f"{address}, {community}".strip(", ") if community else address full_desc = (f"{cat_disp} — {desc}".strip(" —")[:400] if desc else cat_disp[:400]) p = { "municipality": "City of Calgary", "address": full_address[:200], "app_number": app_num, "description": full_desc, "permit_type": infer_permit_type(full_desc), "applicant": applicant[:150], "status": status, "date_submitted": app_date, "date_found": datetime.now().strftime("%Y-%m-%d"), "source_url": (f"https://dmap.calgary.ca/?p={app_num}" if app_num else "https://data.calgary.ca/Business-and-Economic-Activity/Development-Permits/6933-unw5"), "id": make_id("calgary", app_num, address), } if lat and lng: p["lat"] = lat p["lng"] = lng permits.append(p) log(f" Calgary: {len(permits)} qualifying permits from SODA API") except Exception as e: log(f" Calgary SODA parse error: {e}") return permits # ─── MAIN SCRAPER DISPATCHER ────────────────────────────────────────────────── def scrape_municipality(muni): muni_id = muni["id"] muni_name = muni["name"] url = muni["url"] mtype = muni["type"] log(f"Scraping: {muni_name} ({mtype})") try: if mtype == "pdf": pdf_bytes = http_get(url, as_bytes=True) if not pdf_bytes: return [], False current_hash = get_hash(pdf_bytes) last_hash = load_hash(muni_id) if current_hash == last_hash: log(f" {muni_name}: PDF unchanged, skipping parse") return [], False if last_hash is None: log(f" {muni_name}: no saved hash — find_new() will deduplicate against existing DB") save_hash(muni_id, current_hash) if muni_id == "burnaby": permits = parse_burnaby_pdf(pdf_bytes, url) else: permits = parse_pdf_permits(pdf_bytes, muni_name) for p in permits: p["source_url"] = url return permits, True elif mtype == "html_portal": # Langley: hash the FeatureServer response to detect changes raw = http_get(url) if not raw: return [], False current_hash = get_hash(raw) last_hash = load_hash(muni_id) changed = current_hash != last_hash save_hash(muni_id, current_hash) permits = scrape_langley() if changed else [] return permits, changed elif mtype == "arcgis": permits = scrape_arcgis(muni_id, muni_name, url) return permits, True elif mtype == "arcgis_coq": permits = scrape_coquitlam(url) return permits, True elif mtype == "html_weekly": return scrape_chilliwack(), True elif mtype == "civicweb" or mtype == "arcgis_delta": return scrape_delta(), True elif mtype == "arcgis_victoria": return scrape_victoria(), True elif mtype == "arcgis_nanaimo": return scrape_nanaimo(), True elif mtype == "arcgis_squamish": return scrape_squamish(), True elif mtype == "arcgis_kamloops": return scrape_kamloops(), True elif mtype == "html_langley_city": raw = http_get(muni["url"]) if not raw: return [], False current_hash = get_hash(raw) last_hash = load_hash(muni_id) changed = current_hash != last_hash save_hash(muni_id, current_hash) permits = scrape_langley_city() if changed else [] return permits, changed elif mtype in ("escribemeetings_langford", "escribemeetings_kelowna"): permits = _parse_escribemeetings( city_name=muni_name, city_id=muni_id, rss_url=url, muni_url=url, ) return permits, bool(permits) elif mtype == "soda": permits = scrape_calgary(url) return permits, True except Exception as e: log(f" ERROR scraping {muni_name}: {e}") return [], False return [], False # ─── DATABASE ───────────────────────────────────────────────────────────────── def load_db(): if os.path.exists(DB_FILE): with open(DB_FILE, "r", encoding="utf-8") as f: return json.load(f) return {"permits": [], "last_updated": None} def save_db(db): db["last_updated"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") with open(DB_FILE, "w", encoding="utf-8") as f: json.dump(db, f, indent=2, ensure_ascii=False) def find_new(existing, incoming): existing_ids = {p["id"] for p in existing} return [p for p in incoming if p["id"] not in existing_ids] # ─── EXCEL EXPORT ───────────────────────────────────────────────────────────── def sanitize(value): if not isinstance(value, str): return value return re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f]', '', value) def export_excel(permits): try: import openpyxl from openpyxl.styles import Font, PatternFill, Alignment, Border, Side from openpyxl.utils import get_column_letter except ImportError: log(" Excel export skipped — openpyxl not installed (pip3 install openpyxl)") return wb = openpyxl.Workbook() # ── Sheet 1: Summary ───────────────────────────────────────────────────── ws_sum = wb.active ws_sum.title = "Summary" # ── Sheet 2: All Permits ───────────────────────────────────────────────── ws_all = wb.create_sheet("All Permits") columns = ["Municipality", "App Number", "Date Submitted", "Address", "Description", "Status", "Date Found", "Source URL"] col_widths = [22, 14, 15, 32, 60, 14, 13, 45] header_fill = PatternFill("solid", fgColor="0D2240") header_font = Font(name="Arial", bold=True, color="FFFFFF", size=10) for col_idx, (col_name, width) in enumerate(zip(columns, col_widths), 1): cell = ws_all.cell(row=1, column=col_idx, value=col_name) cell.fill = header_fill cell.font = header_font cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) ws_all.column_dimensions[get_column_letter(col_idx)].width = width ws_all.row_dimensions[1].height = 22 muni_colors = {m["name"]: m["color"].replace("#", "") for m in MUNICIPALITIES} thin = Side(style="thin", color="D0DCE8") border = Border(bottom=thin) sorted_permits = sorted(permits, key=lambda p: (p["municipality"], p.get("date_submitted", p.get("date_found", ""))), reverse=True) for row_idx, p in enumerate(sorted_permits, 2): row_data = [ sanitize(p.get("municipality", "")), sanitize(p.get("app_number", "")), sanitize(p.get("date_submitted", "")), sanitize(p.get("address", "")), sanitize(p.get("description", "")), sanitize(p.get("status", "")), sanitize(p.get("date_found", "")), sanitize(p.get("source_url", "")), ] fill_color = "F4F7FC" if row_idx % 2 == 0 else "FFFFFF" muni_name = p.get("municipality", "") muni_hex = muni_colors.get(muni_name, "003366") for col_idx, value in enumerate(row_data, 1): cell = ws_all.cell(row=row_idx, column=col_idx, value=value) cell.border = border cell.fill = PatternFill("solid", fgColor=fill_color) cell.alignment = Alignment(vertical="top", wrap_text=(col_idx == 5)) if col_idx == 1: cell.font = Font(name="Arial", size=9, bold=True, color=muni_hex) elif col_idx == 4: cell.font = Font(name="Arial", size=9, bold=True, color="0D2240") else: cell.font = Font(name="Arial", size=9) ws_all.row_dimensions[row_idx].height = 30 ws_all.freeze_panes = "A2" # ── Summary sheet ───────────────────────────────────────────────────────── ws_sum.column_dimensions["A"].width = 26 ws_sum.column_dimensions["B"].width = 16 ws_sum.column_dimensions["C"].width = 20 ws_sum.column_dimensions["D"].width = 20 ws_sum.merge_cells("A1:D1") title_cell = ws_sum["A1"] title_cell.value = "Development Permits — Summary by Municipality (BC + Calgary)" title_cell.font = Font(name="Arial", bold=True, size=14, color="0D2240") title_cell.alignment = Alignment(horizontal="center", vertical="center") ws_sum.row_dimensions[1].height = 32 ws_sum["A2"].value = f"Generated: {datetime.now().strftime('%B %d, %Y at %I:%M %p')}" ws_sum["A2"].font = Font(name="Arial", italic=True, size=9, color="607080") ws_sum.row_dimensions[2].height = 18 sum_headers = ["Municipality", "Total Permits", "New This Week", "Last Entry Date"] for col_idx, h in enumerate(sum_headers, 1): cell = ws_sum.cell(row=4, column=col_idx, value=h) cell.fill = PatternFill("solid", fgColor="0D2240") cell.font = Font(name="Arial", bold=True, color="FFFFFF", size=10) cell.alignment = Alignment(horizontal="center") ws_sum.row_dimensions[4].height = 20 one_week_ago = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d") muni_order = [m["name"] for m in MUNICIPALITIES] for row_idx, muni_name in enumerate(muni_order, 5): muni_permits = [p for p in permits if p.get("municipality") == muni_name] new_count = sum(1 for p in muni_permits if p.get("date_found", "") >= one_week_ago) dates = [p.get("date_found", "") for p in muni_permits if p.get("date_found")] last_date = max(dates) if dates else "—" muni_hex = muni_colors.get(muni_name, "003366") fill_color = "F4F7FC" if row_idx % 2 == 0 else "FFFFFF" for col_idx, val in enumerate([muni_name, len(muni_permits), new_count, last_date], 1): cell = ws_sum.cell(row=row_idx, column=col_idx, value=val) cell.fill = PatternFill("solid", fgColor=fill_color) cell.alignment = Alignment(horizontal="center" if col_idx > 1 else "left", vertical="center") cell.border = border if col_idx == 1: cell.font = Font(name="Arial", bold=True, color=muni_hex, size=10) elif col_idx == 3 and new_count > 0: cell.font = Font(name="Arial", bold=True, color="0E8A6E", size=10) else: cell.font = Font(name="Arial", size=10) ws_sum.row_dimensions[row_idx].height = 22 total_row = len(muni_order) + 5 new_total = sum(1 for p in permits if p.get("date_found", "") >= one_week_ago) for col_idx, val in enumerate(["TOTAL", len(permits), new_total, ""], 1): cell = ws_sum.cell(row=total_row, column=col_idx, value=val) cell.fill = PatternFill("solid", fgColor="0D2240") cell.font = Font(name="Arial", bold=True, color="FFFFFF", size=10) cell.alignment = Alignment(horizontal="center" if col_idx > 1 else "left", vertical="center") # ── Per-municipality sheets ─────────────────────────────────────────────── BURNABY_COLS = ["Municipality", "Plan Areas", "Type", "App Number", "Developer/Architect", "Address", "Description", "Residential Strata Units", "Residential Rental Units", "Status", "Date Found", "Source URL"] BURNABY_WIDTHS = [20, 18, 28, 14, 32, 36, 60, 10, 10, 16, 13, 45] for muni in MUNICIPALITIES: muni_name = muni["name"] muni_hex = muni["color"].replace("#", "") sheet_name = muni_name.replace("City of ", "").replace("Township of ", "")[:31] ws = wb.create_sheet(sheet_name) is_burnaby = (muni_name == "City of Burnaby") sheet_cols = BURNABY_COLS if is_burnaby else columns sheet_widths = BURNABY_WIDTHS if is_burnaby else col_widths for col_idx, (col_name, width) in enumerate(zip(sheet_cols, sheet_widths), 1): cell = ws.cell(row=1, column=col_idx, value=col_name) cell.fill = PatternFill("solid", fgColor=muni_hex) cell.font = Font(name="Arial", bold=True, color="FFFFFF", size=10) cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) ws.column_dimensions[get_column_letter(col_idx)].width = width ws.row_dimensions[1].height = 22 muni_permits = [p for p in sorted_permits if p.get("municipality") == muni_name] for row_idx, p in enumerate(muni_permits, 2): if is_burnaby: row_data = [ sanitize(p.get("municipality", "")), sanitize(p.get("plan_area", "")), sanitize(p.get("permit_type", "")), sanitize(p.get("app_number", "")), sanitize(p.get("developer", "")), sanitize(p.get("address", "")), sanitize(p.get("description", "")), sanitize(str(p.get("strata_units", ""))), sanitize(str(p.get("rental_units", ""))), sanitize(p.get("status", "")), sanitize(p.get("date_found", "")), sanitize(p.get("source_url", "")), ] else: row_data = [ sanitize(p.get("municipality", "")), sanitize(p.get("app_number", "")), sanitize(p.get("date_submitted", "")), sanitize(p.get("address", "")), sanitize(p.get("description", "")), sanitize(p.get("status", "")), sanitize(p.get("date_found", "")), sanitize(p.get("source_url", "")), ] fill_color = "F4F7FC" if row_idx % 2 == 0 else "FFFFFF" for col_idx, value in enumerate(row_data, 1): cell = ws.cell(row=row_idx, column=col_idx, value=value) cell.border = border cell.fill = PatternFill("solid", fgColor=fill_color) cell.alignment = Alignment(vertical="top", wrap_text=(col_idx == 7 if is_burnaby else col_idx == 5)) cell.font = Font(name="Arial", size=9) ws.row_dimensions[row_idx].height = 30 ws.freeze_panes = "A2" wb.save(EXCEL_FILE) log(f"Excel saved: {EXCEL_FILE} ({len(permits)} permits, {len(wb.sheetnames)} sheets)") # ─── DASHBOARD ──────────────────────────────────────────────────────────────── def generate_dashboard(permits): permits_json = json.dumps(permits, ensure_ascii=False) muni_list = sorted(set(p["municipality"] for p in permits)) muni_list_json = json.dumps(muni_list) muni_colors_json = json.dumps({m["name"]: m["color"] for m in MUNICIPALITIES}) team_members_json = json.dumps(CONFIG.get("team_members", [])) monday_board_id = CONFIG.get("monday", {}).get("board_id", "") total = len(permits) new_count = sum(1 for p in permits if p.get("date_found") and (datetime.now() - datetime.strptime(p["date_found"], "%Y-%m-%d")).days <= 7) updated = datetime.now().strftime("%B %d, %Y at %I:%M %p") # Load Leaflet from same folder — embed so Chrome file:// works with no network # Skip entirely when GENERATE_MAP is False to keep the dashboard lightweight if GENERATE_MAP: script_dir = os.path.dirname(os.path.abspath(__file__)) leaflet_js_path = os.path.join(script_dir, "leaflet.js") leaflet_css_path = os.path.join(script_dir, "leaflet.css") leaflet_js = open(leaflet_js_path).read() if os.path.exists(leaflet_js_path) else "console.warn('leaflet.js missing');" leaflet_css = open(leaflet_css_path).read() if os.path.exists(leaflet_css_path) else "" else: leaflet_js = "" leaflet_css = "" # Pre-compute map-conditional HTML fragments (avoids f-string backslash issues) map_tab_btn = '' if GENERATE_MAP else "" map_view_html = """
🔍
Loading map…
""" if GENERATE_MAP else "" map_tab_js = """ let mapInitialized = false; let leafletMap = null; let markerCluster = null; let allMarkers = []; """ if GENERATE_MAP else "" map_showtab_extras = """ document.getElementById('tab-map').classList.toggle('active', tab === 'map'); document.getElementById('view-map').style.display = tab === 'map' ? 'block' : 'none'; if (tab === 'map' && !mapInitialized) initMap();""" if GENERATE_MAP else "" map_filter_js = ( "// Populate map filters (mirrors)\n" "const mapMuniSel = document.getElementById('mapFilterMuni');\n" "MUNIS.forEach(m => { const o=document.createElement('option'); o.value=m; o.textContent=m; mapMuniSel.appendChild(o); });\n" "const mapTypeSel = document.getElementById('mapFilterType');\n" "allTypes.forEach(t => { const o=document.createElement('option'); o.value=t; o.textContent=t; mapTypeSel.appendChild(o); });" ) if GENERATE_MAP else "// Map filters skipped (GENERATE_MAP=False)" # initMap + filterMapMarkers — needs muni_colors_json interpolated, so pre-compute here if GENERATE_MAP: map_functions_js = f""" function initMap() {{ mapInitialized = true; leafletMap = L.map('leaflet-map').setView([49.15, -122.85], 10); L.tileLayer('https://{{s}}.basemaps.cartocdn.com/light_all/{{z}}/{{x}}/{{y}}{{r}}.png', {{ attribution: '© OpenStreetMap © CARTO', subdomains: 'abcd', maxZoom: 19 }}).addTo(leafletMap); const mappable = ALL.filter(p => p.lat && p.lng); const unmapped = ALL.length - mappable.length; document.getElementById('map-stats').textContent = mappable.length.toLocaleString() + ' permits mapped' + (unmapped > 0 ? ' · ' + unmapped + ' without coordinates yet' : ''); const COLORS_MAP = {muni_colors_json}; const leads = loadLeads(); mappable.forEach(p => {{ const color = COLORS_MAP[p.municipality] || '#003366'; const addr = p.address || p.site_address || ''; const typ = p.folder_type || p.permit_type || ''; const dev = p.developer || p.applicant || ''; const muniShort = p.municipality.replace('City of ','').replace('Township of ',''); const srcLink = p.source_url ? `View source ↗` : ''; const newFlag = p.date_found && Math.floor((new Date() - new Date(p.date_found)) / 86400000) <= 7; const isLead = leads[p.id] === 'lead'; const pinColor = getTypeColor(typ); const icon = isLead ? L.divIcon({{ html: '
⭐
', className: '', iconSize: [24,24], iconAnchor: [12,12], popupAnchor: [0,-14] }}) : L.divIcon({{ html: `
`, className: '', iconSize: [12,12], iconAnchor: [6,6], popupAnchor: [0,-8] }}); const marker = L.marker([p.lat, p.lng], {{icon}}); marker.bindPopup(`
${{muniShort}}
${{addr || '—'}}
${{newFlag ? '
● NEW THIS WEEK
' : ''}}
${{typ || ''}}
${{p.app_number ? `
${{p.app_number}}
` : ''}}
${{(p.description||'').substring(0,200)}}${{(p.description||'').length > 200 ? '…' : ''}}
${{dev ? `
Developer: ${{dev}}
` : ''}}
${{p.date_found || ''}} ${{srcLink}}
`, {{maxWidth: 340}}); allMarkers.push([marker, p]); marker.addTo(leafletMap); }}); document.getElementById('mapFilterMuni').addEventListener('change', filterMapMarkers); document.getElementById('mapFilterType').addEventListener('change', filterMapMarkers); document.getElementById('mapFilterDate').addEventListener('change', filterMapMarkers); document.getElementById('mapSearch').addEventListener('input', filterMapMarkers); }} function filterMapMarkers() {{ if (!leafletMap) return; const q = document.getElementById('mapSearch').value.toLowerCase(); const muni = document.getElementById('mapFilterMuni').value; const type = document.getElementById('mapFilterType').value; const days = parseInt(document.getElementById('mapFilterDate').value) || 0; let shown = 0; allMarkers.forEach(item => {{ const marker = item[0], p = item[1]; const addr = p.address || p.site_address || ''; const typ = p.folder_type || p.permit_type || ''; const dev = p.developer || p.applicant || ''; const txt = (addr+' '+(p.app_number||'')+' '+(p.description||'')+' '+dev+' '+p.municipality).toLowerCase(); const show = (!q || txt.includes(q)) && (!muni || p.municipality === muni) && (!type || typ === type) && (!days || (p.date_found && Math.floor((new Date()-new Date(p.date_found))/86400000) <= days)); if (show) {{ marker.addTo(leafletMap); shown++; }} else {{ marker.remove(); }} }}); document.getElementById('map-stats').textContent = shown.toLocaleString() + ' permits shown'; }} function goToMap(lat, lng, id) {{ showTab('map'); const fly = () => {{ leafletMap.flyTo([lat, lng], 17, {{duration: 0.8}}); const entry = allMarkers.find(([m, p]) => p.id === id); if (entry) setTimeout(() => entry[0].openPopup(), 900); }}; mapInitialized ? fly() : setTimeout(fly, 150); }} """ else: map_functions_js = "" html = f""" Orion Permit Tracking Dashboard

🏗 Orion Permit Tracking Dashboard

Updated {updated}
{total}
Total Permits
{new_count}
New This Week
{total}
Showing
{len(muni_list)}
Municipalities
{map_tab_btn}
🔍
Lead? Score ↕ Muni ↕ App # ↕ Address ↕ Type Description Developer Status ↕ Date Found ↕ Source Map
{map_view_html}
⭐ Marked Leads
Lead? Score Muni App # Address Type Description Developer Date Found Source Map Pipeline
Sourced from official municipal websites · Surrey · Langley · Abbotsford · Chilliwack · Burnaby · Richmond · Coquitlam · Delta · Calgary
""" with open(DASHBOARD_FILE, "w", encoding="utf-8") as f: f.write(html) log(f"Dashboard updated: {DASHBOARD_FILE}") # ─── EMAIL ALERT ────────────────────────────────────────────────────────────── def send_email(new_permits, total): cfg = CONFIG["email"] if not cfg["enabled"] or not new_permits: return count = len(new_permits) subject = (f"🏗️ BC Permit Alert: {count} New Development Permit{'s' if count>1 else ''} " f"Across {len(set(p['municipality'] for p in new_permits))} Municipality/Municipalities") by_muni = {} for p in new_permits: by_muni.setdefault(p["municipality"], []).append(p) sections = "" for muni_name, ps in sorted(by_muni.items()): muni_color = next((m["color"] for m in MUNICIPALITIES if m["name"] == muni_name), "#003366") rows = "".join(f""" {p['address'] or '—'} {p.get('app_number','—')} {p.get('description','—')[:120]} {p['date_found']} """ for p in ps) sections += f"""

{muni_name} — {len(ps)} new permit{'s' if len(ps)>1 else ''}

{rows}
AddressApp #DescriptionDate Found
""" html_body = f"""

🏗️ BC Development Permit Weekly Alert

{count} new permit{'s' if count>1 else ''} detected across {len(by_muni)} municipality/municipalities  ·  {datetime.now().strftime('%B %d, %Y')}

{sections}

Total permits in database: {total}  |  9 municipalities monitored (BC + Calgary)

""" try: msg = MIMEMultipart("alternative") msg["Subject"] = subject msg["From"] = cfg["sender_email"] msg["To"] = ", ".join(cfg["recipient_emails"]) msg.attach(MIMEText(html_body, "html")) if os.path.exists(EXCEL_FILE): with open(EXCEL_FILE, "rb") as f: part = MIMEBase("application", "octet-stream") part.set_payload(f.read()) encoders.encode_base64(part) part.add_header("Content-Disposition", "attachment; filename=bc_permits.xlsx") msg.attach(part) with smtplib.SMTP(cfg["smtp_server"], cfg["smtp_port"]) as server: server.ehlo(); server.starttls() server.login(cfg["sender_email"], cfg["sender_password"]) server.sendmail(cfg["sender_email"], cfg["recipient_emails"], msg.as_string()) log(f"Email sent: {count} new permits to {cfg['recipient_emails']}") except Exception as e: log(f"Email send error: {e}") # ─── GEOCODER ───────────────────────────────────────────────────────────────── # Uses Nominatim (OpenStreetMap) — free, no API key required. # Geocodes any permit missing lat/lng, caches results to avoid repeat lookups. GEOCODE_CACHE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "hashes", "geocode_cache.json") def load_geocode_cache(): if os.path.exists(GEOCODE_CACHE_FILE): with open(GEOCODE_CACHE_FILE, "r", encoding="utf-8") as f: return json.load(f) return {} def save_geocode_cache(cache): os.makedirs(os.path.dirname(GEOCODE_CACHE_FILE), exist_ok=True) with open(GEOCODE_CACHE_FILE, "w", encoding="utf-8") as f: json.dump(cache, f, indent=2) def geocode_address(raw_address, city_hint, cache): """Look up lat/lng for an address. Returns (lat, lng) or (None, None). Uses BC Address Geocoder for BC municipalities (fast, no strict rate limit), falls back to Nominatim for Calgary (AB).""" if not raw_address or len(raw_address) < 5: return None, None # Clean address addr = re.sub(r'\s+', ' ', raw_address).strip() addr = re.split(r'\b(Development Permit|Heritage|Rezoning|Subdivision)\b', addr)[0].strip() addr = addr.rstrip(',').strip() cache_key = f"{addr}|{city_hint}" if cache_key in cache: return cache[cache_key].get("lat"), cache[cache_key].get("lng") lat, lng = None, None if city_hint == "Calgary": # Calgary (Alberta) — use Nominatim try: params = urllib.parse.urlencode({"q": f"{addr}, Calgary, AB, Canada", "format": "json", "limit": 1, "countrycodes": "ca"}) req = urllib.request.Request( f"https://nominatim.openstreetmap.org/search?{params}", headers={"User-Agent": "OrionPermitTracker/1.0"}) with urllib.request.urlopen(req, timeout=8) as resp: results = json.loads(resp.read().decode("utf-8")) if results: lat, lng = float(results[0]["lat"]), float(results[0]["lon"]) except Exception: pass else: # BC municipalities — use BC Address Geocoder (government API, faster) try: params = urllib.parse.urlencode({ "addressString": f"{addr}, {city_hint}, BC", "maxResults": 1, "outputSRS": 4326, }) req = urllib.request.Request( f"https://geocoder.api.gov.bc.ca/addresses.json?{params}", headers={"User-Agent": "OrionPermitTracker/1.0"}) with urllib.request.urlopen(req, timeout=8) as resp: data = json.loads(resp.read().decode("utf-8")) features = data.get("features", []) if features: score = features[0].get("properties", {}).get("score", 0) if score >= 50: coords = features[0]["geometry"]["coordinates"] cand_lng, cand_lat = float(coords[0]), float(coords[1]) bounds = MUNI_BOUNDS.get(city_hint) if bounds: lat_ok = bounds["lat"][0] <= cand_lat <= bounds["lat"][1] lng_ok = bounds["lng"][0] <= cand_lng <= bounds["lng"][1] if lat_ok and lng_ok: lat, lng = cand_lat, cand_lng else: log(f" Geocoder OOB: {addr!r} ({city_hint}) score={score} " f"({cand_lat:.4f},{cand_lng:.4f}) rejected") else: lat, lng = cand_lat, cand_lng except Exception: pass cache[cache_key] = {"lat": lat, "lng": lng} return lat, lng # BC municipalities use faster gov geocoder; Calgary uses Nominatim (1s rate limit) BC_CITIES = {"Surrey", "Langley", "Abbotsford", "Chilliwack", "Burnaby", "Richmond", "Coquitlam", "Delta", "Victoria", "Nanaimo", "Squamish", "Kamloops", "Langford", "Kelowna"} # Bounding boxes for geocode validation — rejects results outside expected area MUNI_BOUNDS = { "Surrey": {"lat": (49.00, 49.25), "lng": (-122.97, -122.65)}, "Langley": {"lat": (49.00, 49.20), "lng": (-122.75, -122.45)}, "Burnaby": {"lat": (49.19, 49.30), "lng": (-123.07, -122.85)}, "Richmond": {"lat": (49.08, 49.22), "lng": (-123.28, -123.02)}, "Coquitlam": {"lat": (49.20, 49.45), "lng": (-122.90, -122.62)}, "Chilliwack": {"lat": (49.00, 49.22), "lng": (-122.15, -121.70)}, "Delta": {"lat": (49.00, 49.20), "lng": (-123.25, -122.82)}, "Abbotsford": {"lat": (49.00, 49.22), "lng": (-122.45, -122.05)}, "Calgary": {"lat": (50.80, 51.30), "lng": (-114.40, -113.80)}, "Victoria": {"lat": (48.38, 48.50), "lng": (-123.45, -123.28)}, "Nanaimo": {"lat": (49.10, 49.32), "lng": (-124.10, -123.92)}, "Squamish": {"lat": (49.60, 49.88), "lng": (-123.22, -123.05)}, "Kamloops": {"lat": (50.55, 50.80), "lng": (-120.55, -120.20)}, "Langford": {"lat": (48.40, 48.50), "lng": (-123.55, -123.45)}, "Kelowna": {"lat": (49.78, 50.03), "lng": (-119.62, -119.33)}, } def clean_bad_coordinates(db): """Remove out-of-bounds lat/lng from geocode cache and permits database. This forces bad addresses to be re-geocoded on the next run.""" city_map = { "City of Surrey": "Surrey", "Township of Langley": "Langley", "City of Langley": "Langley", "City of Abbotsford": "Abbotsford", "City of Chilliwack": "Chilliwack", "City of Burnaby": "Burnaby", "City of Richmond": "Richmond", "City of Coquitlam": "Coquitlam", "City of Delta": "Delta", "City of Calgary": "Calgary", "City of Kamloops": "Kamloops", "City of Langford": "Langford", "City of Kelowna": "Kelowna", } def in_bounds(city, lat, lng): b = MUNI_BOUNDS.get(city) if not b: return True # unknown city — don't touch return (b["lat"][0] <= lat <= b["lat"][1] and b["lng"][0] <= lng <= b["lng"][1]) # 1. Clean geocode cache cache = load_geocode_cache() cache_nulled = 0 for key in list(cache.keys()): entry = cache[key] lat, lng = entry.get("lat"), entry.get("lng") if lat is None or lng is None: continue city = key.split("|")[-1] if "|" in key else "" if not in_bounds(city, lat, lng): cache[key] = {"lat": None, "lng": None} cache_nulled += 1 if cache_nulled: save_geocode_cache(cache) # 2. Clear out-of-bounds coords from permits so they get re-geocoded db_cleared = 0 for p in db.get("permits", []): lat = p.get("lat") lng = p.get("lng") if lat is None or lng is None: continue city = city_map.get(p.get("municipality", ""), "") if city and not in_bounds(city, lat, lng): p.pop("lat", None) p.pop("lng", None) db_cleared += 1 log(f" Coordinate cleanup: nulled {cache_nulled} cache entries, " f"cleared {db_cleared} permit coordinates for re-geocoding") def geocode_permits(permits, db=None, db_path=None): """Add lat/lng to any permits that don't have them yet. Saves progress to disk every 50 permits so a killed job doesn't lose work.""" cache = load_geocode_cache() needs_geocoding = [p for p in permits if not p.get("lat")] if not needs_geocoding: return log(f" Geocoding {len(needs_geocoding)} permits...") geocoded = 0 city_map = { "City of Surrey": "Surrey", "Township of Langley": "Langley", "City of Langley": "Langley", "City of Abbotsford": "Abbotsford", "City of Chilliwack": "Chilliwack", "City of Burnaby": "Burnaby", "City of Richmond": "Richmond", "City of Coquitlam": "Coquitlam", "City of Delta": "Delta", "City of Calgary": "Calgary", "City of Victoria": "Victoria", "City of Nanaimo": "Nanaimo", "District of Squamish": "Squamish", "City of Kamloops": "Kamloops", "City of Langford": "Langford", "City of Kelowna": "Kelowna", } for i, p in enumerate(needs_geocoding): addr = p.get("address") or p.get("site_address") or "" city = city_map.get(p.get("municipality", ""), "") lat, lng = geocode_address(addr, city, cache) if lat and lng: p["lat"] = lat p["lng"] = lng geocoded += 1 # Rate limit: BC geocoder is fast; Nominatim needs 1.1s time.sleep(0.08 if city in BC_CITIES else 1.1) # Save progress every 50 permits so a killed job doesn't lose work if (i + 1) % 50 == 0: save_geocode_cache(cache) if db is not None and db_path: with open(db_path, "w", encoding="utf-8") as f: json.dump(db, f, indent=2, ensure_ascii=False) log(f" ...{i+1}/{len(needs_geocoding)} geocoded so far ({geocoded} hits)") save_geocode_cache(cache) log(f" Geocoded {geocoded} of {len(needs_geocoding)} permits") # ─── LEADS FILE HELPERS ─────────────────────────────────────────────────────── LEADS_FILE = os.path.join(DATA_DIR, "leads.json") def load_leads_file(): """Load leads.json, migrating flat {id:'lead'} format to object format.""" if not os.path.exists(LEADS_FILE): return {} with open(LEADS_FILE, encoding="utf-8") as f: data = json.load(f) migrated = {} changed = False for pid, val in data.items(): if isinstance(val, str): migrated[pid] = { "status": val, "stage": None, "notes": "", "assigned_to": "", "contacted_date": None, "monday_item_id": None, } changed = True else: migrated[pid] = val if changed: with open(LEADS_FILE, "w", encoding="utf-8") as f: json.dump(migrated, f, indent=2) return migrated def save_leads_file(leads): with open(LEADS_FILE, "w", encoding="utf-8") as f: json.dump(leads, f, indent=2) # ─── MONDAY.COM INTEGRATION ─────────────────────────────────────────────────── def monday_create_item(permit): """POST a permit to Monday.com as a new board item. Returns item_id or None.""" cfg = CONFIG.get("monday", {}) if not cfg.get("enabled") or not cfg.get("api_token") or not cfg.get("board_id"): return None col_map = cfg.get("column_map", {}) col_values = {} for col_id, field_name in col_map.items(): val = permit.get(field_name, "") if not val: continue if col_id.startswith("date"): col_values[col_id] = {"date": str(val)[:10]} elif col_id == "link": col_values[col_id] = {"url": str(val), "text": "View Permit"} elif col_id.startswith("numbers"): col_values[col_id] = str(val) else: col_values[col_id] = str(val)[:500] muni_short = permit.get("municipality", "").replace("City of ", "").replace("Township of ", "") addr = permit.get("address") or permit.get("site_address") or permit.get("app_number") or "Unknown" item_name = f"{muni_short} — {addr}"[:255] query = """mutation ($board: ID!, $group: String!, $name: String!, $cols: JSON!) { create_item(board_id: $board, group_id: $group, item_name: $name, column_values: $cols) { id } }""" variables = { "board": cfg["board_id"], "group": cfg.get("group_id", "new_group"), "name": item_name, "cols": json.dumps(col_values), } payload = json.dumps({"query": query, "variables": variables}).encode("utf-8") req = urllib.request.Request( "https://api.monday.com/v2", data=payload, headers={ "Content-Type": "application/json", "Authorization": cfg["api_token"], "API-Version": "2024-01", }, method="POST", ) try: with urllib.request.urlopen(req, timeout=15) as resp: result = json.loads(resp.read().decode("utf-8")) item_id = result.get("data", {}).get("create_item", {}).get("id") if item_id: log(f" Monday.com: created item {item_id} for {item_name[:60]}") return item_id log(f" Monday.com: API error — {result.get('errors', result)}") return None except Exception as e: log(f" Monday.com: request failed — {e}") return None def push_new_permits_to_monday(new_permits): """Auto-push new permits above score threshold to Monday.com board.""" cfg = CONFIG.get("monday", {}) if not cfg.get("enabled"): return threshold = cfg.get("auto_score_threshold", 60) leads = load_leads_file() pushed = 0 for p in new_permits: if (p.get("score", 0) or 0) < threshold: continue pid = p["id"] existing = leads.get(pid, {}) if isinstance(existing, dict) and existing.get("monday_item_id"): continue # already in Monday item_id = monday_create_item(p) if item_id: leads[pid] = { "status": "lead", "stage": "Uncontacted", "notes": "", "assigned_to": "", "contacted_date": None, "monday_item_id": item_id, } pushed += 1 if pushed: save_leads_file(leads) log(f" Monday.com: pushed {pushed} high-score permits to board") # ─── SLACK ALERTS ───────────────────────────────────────────────────────────── def send_slack_alerts(new_permits): """Send Slack Block Kit messages for new permits above score threshold.""" cfg = CONFIG.get("slack", {}) if not cfg.get("enabled") or not cfg.get("webhook_url"): return threshold = cfg.get("score_threshold", 60) type_filter = cfg.get("permit_types", []) qualifying = [ p for p in new_permits if (p.get("score", 0) or 0) >= threshold and (not type_filter or p.get("permit_type") in type_filter) ] if not qualifying: return for p in qualifying: score = p.get("score", 0) or 0 emoji = "🔥" if score >= 80 else "⭐" if score >= 60 else "📋" muni_short = p.get("municipality", "").replace("City of ", "").replace("Township of ", "") addr = p.get("address") or p.get("site_address") or "Unknown address" ptype = p.get("permit_type") or "—" desc = (p.get("description") or "")[:200] src = p.get("source_url", "") src_link = f"<{src}|View permit ↗>" if src else "" dev = p.get("developer") or p.get("applicant") or "" dev_line = f"\n*Developer:* {dev}" if dev else "" blocks = [ {"type": "header", "text": {"type": "plain_text", "text": f"{emoji} New {ptype} — {muni_short} (Score: {score}/100)"}}, {"type": "section", "fields": [ {"type": "mrkdwn", "text": f"*Address:*\n{addr}"}, {"type": "mrkdwn", "text": f"*Score:*\n{score}/100"}, {"type": "mrkdwn", "text": f"*Type:*\n{ptype}"}, {"type": "mrkdwn", "text": f"*Municipality:*\n{muni_short}"}, ]}, {"type": "section", "text": {"type": "mrkdwn", "text": f"*Description:* {desc}{dev_line}"}}, {"type": "context", "elements": [{"type": "mrkdwn", "text": f"Found: {p.get('date_found','')} · App#: {p.get('app_number','')} · {src_link}"}]}, ] payload = json.dumps({"blocks": blocks}).encode("utf-8") req = urllib.request.Request( cfg["webhook_url"], data=payload, headers={"Content-Type": "application/json"}, method="POST", ) try: urllib.request.urlopen(req, timeout=10) log(f" Slack: sent alert for {addr[:50]} (score={score})") except Exception as e: log(f" Slack: webhook failed — {e}") # ─── MAIN ───────────────────────────────────────────────────────────────────── def run_agent(): log("=" * 65) log("Orion Permit Agent starting (BC + Calgary)...") log(f"Monitoring {len(MUNICIPALITIES)} municipalities") db = load_db() all_new = [] for muni in MUNICIPALITIES: time.sleep(1) incoming, changed = scrape_municipality(muni) if changed and incoming: new = find_new(db["permits"], incoming) log(f" → {len(new)} new permits for {muni['name']} ({len(incoming)} total scraped)") all_new.extend(new) db["permits"].extend(new) elif not changed: log(f" → {muni['name']}: no changes detected") else: log(f" → {muni['name']}: no permits parsed (site may be unavailable)") # ── Backfill pass: types, Surrey PLR URLs, Calgary dmap URLs ────────────── type_backfilled = url_backfilled = 0 OLD_CALGARY_URL = "https://data.calgary.ca/Business-and-Economic-Activity/Development-Permits/6933-unw5" for p in db["permits"]: # Normalise / infer permit_type for every permit (handles Burnaby verbose labels too) old_type = p.get("permit_type", "") new_type = infer_permit_type(p.get("description", ""), old_type) if new_type and new_type != old_type: p["permit_type"] = new_type type_backfilled += 1 # Calgary: replace dataset page URL with direct dmap link if "Calgary" in p.get("municipality", "") and p.get("source_url", "") == OLD_CALGARY_URL: app = p.get("app_number", "") if app: p["source_url"] = f"https://dmap.calgary.ca/?p={app}" url_backfilled += 1 if type_backfilled: log(f" Normalised permit_type for {type_backfilled} permits") if url_backfilled: log(f" Backfilled direct source URLs for {url_backfilled} permits") # Score every permit (re-run each pass so recency bonus decays correctly) for p in db["permits"]: p["score"] = score_permit(p) log(f" Scored {len(db['permits'])} permits") # Remove any existing coordinates that fall outside city bounding boxes clean_bad_coordinates(db) # Geocode any permits missing coordinates geocode_permits(db["permits"], db=db, db_path=DB_FILE) save_db(db) log(f"Database: {len(db['permits'])} total permits across all municipalities") export_excel(db["permits"]) generate_dashboard(db["permits"]) if all_new: log(f"Sending email alert for {len(all_new)} new permits...") send_email(all_new, len(db["permits"])) push_new_permits_to_monday(all_new) send_slack_alerts(all_new) else: log("No new permits this run — no email sent.") log("Agent run complete.") log("=" * 65) if __name__ == "__main__": run_agent()