#!/usr/bin/env python
# creator:  Silas Jelley
# created: 2020-08-11 09:52:32
# updated: 2023-09-21 08:52:49
# version: 2.5

# Profiling
import time
import types

start = time.time()


def profbegin():
    global profstart
    profstart = time.time()


def profend(profmessage):
    profstop = time.time()
    profduration = round((profstop - profstart), 3)
    proftime = time.strftime("%H:%M:%S", time.localtime(profstop))
    print(f"{proftime} {profmessage} in {profduration} seconds")


# Imports
profbegin()
# Standard
import os, re, sys, random, json, pickle, datetime
from subprocess import run, CompletedProcess
from hashlib import md5
from pathlib import Path
from shutil import copyfile, rmtree

# Dependencies:
# For fonts (external): fonttools, brotli
# Imports
import frontmatter
from jinja2 import Environment, FileSystemLoader
from PIL import Image, ImageOps
from pillow_heif import register_heif_opener

register_heif_opener()


# Site details
site = {
    "name": "Silas Jelley",
    "created": "2020-08-20",
    "url": "https://silasjelley.com/",
    "baseurl": "silasjelley.com",
    "uid": "d0b81a46-733e-44b1-8181-61cec5681440",
    "description": "An unremarkable man in a breathtaking world",
    "creator": {
        "name": "Silas Jelley",
        "email": "reply@silasjelley.com",
        "born": "1996-06-09",
    },
    "backlinks": 0,
    "wordcount": 0,
    "wordcountWithoutReferences": 0,
    "pagecount": 0,
    "references": 0,
    "primaries": [],
    "secondaries": [],
    "tags": [],
    "data": {},
}

# Variables
# Source paths
assetsDir = "assets/"
feedTemplate = "feed"
feedXlsTemplate = "feed-xsl"
sitemapTemplate = "sitemap"
defaultTemplate = "default"
templateDir = "assets/templates/"
imagesDir = "assets/images/"
stylesheet = "assets/styles.css"
buildDepth = "production"
draftsFile = "drafts/list"
# Outputs
buildType = os.getenv("SITE_BUILD_TYPE", "normal")
if buildType == "drafts":
    outputDir = "/tmp/drafts.silasjelley.com/"
else:
    outputDir = "/tmp/silasjelley.com/"
# Hash stylesheet for cachebusting
with open(stylesheet, "rb") as fileToHash:
    # read contents of the file
    filedata = fileToHash.read()
    # hash contents of the file through
    site["stylesheet_hash"] = md5(filedata).hexdigest()

assets = {}
assetManifests = list(Path("/home/silas/notes/assets/manifests").glob("**/*.json"))
for manifest in assetManifests:
    assets.update(json.load(open(manifest)))

interlinkCount = 0


# OS Environment
try:
    verbosity = int(os.environ["SITE_BUILD_VERBOSITY"])
except KeyError:
    verbosity = 1

# Read BUILD and CONTENT directories from environment, change working dir
journalDirectory = os.environ["JOURNAL_DIR"]
notesDirectory = os.environ["NOTES_DIR"]
buildDirectory = os.environ["SITE_BUILD_DIR"]
startDirectory = os.getcwd()
os.chdir(buildDirectory)

# Jinja Environment
file_loader = FileSystemLoader(templateDir)
env = Environment(loader=file_loader)


def filter_hash(s):
    return


env.filters["hash"] = filter_hash


def filter_shuffle(seq):
    try:
        result = list(seq)
        random.shuffle(result)
        return result
    except:
        return seq


env.filters["shuffle"] = filter_shuffle


def time_local(value, format="%-I:%M%p"):
    return value.strftime(format).lower()


env.filters["time_local"] = time_local


def date_year(value, format="%Y"):
    return value.strftime(format)


env.filters["year"] = date_year


def date_month(value, format="%m"):
    return value.strftime(format)


env.filters["month"] = date_month


def date_day(value, format="%d"):
    return value.strftime(format)


env.filters["day"] = date_day


def date_year_month(value, format="%Y/%m"):
    return value.strftime(format)


env.filters["year_month"] = date_year_month


def date_year_month_day(value, format="%Y/%m/%d"):
    return value.strftime(format)


env.filters["year_month_day"] = date_year_month_day


def date_long_short_month(value, format="%b %e, %Y"):
    return value.strftime(format)


env.filters["date_long_short_month"] = date_long_short_month


def datetime_w3c(value, format="%Y-%m-%dT%H:%M:%S"):
    return value.strftime(format)


env.filters["datetime_w3c"] = datetime_w3c


def date_long_full_month(value, format="%B %e, %Y"):
    return value.strftime(format)


env.filters["date_long_full_month"] = date_long_full_month


def timedate_long(value, format="%-I:%M%p %B %e, %Y"):
    return value.strftime(format)


env.filters["timedate_long"] = timedate_long


def getFiles():
    fileList = []
    matches = 0
    if buildType == "drafts":
        with open(draftsFile) as draftsList:
            checkList = draftsList.read()
            fileList = checkList.splitlines()
            print(fileList)
    else:
        checkList = list(Path(notesDirectory).glob("**/*.md"))
        for filename in checkList:
            availableMatch = ""
            with open(filename, "r") as inputfile:
                filedata = inputfile.read()
                availableMatch = re.search("^available:", filedata, flags=re.MULTILINE)
                if availableMatch:
                    fileList.append(filename)
    return fileList


def processImage(inputImage, outputWidth):
    im = ImageOps.exif_transpose(Image.open(inputImage))
    outputHeight = int(im.size[1] * float((outputWidth / float(im.size[0]))))
    outputImage = im.resize((outputWidth, outputHeight), Image.Resampling.LANCZOS)
    return {"image": outputImage, "height": outputHeight, "width": outputWidth}


def processAssets():
    # Process Asset Manifest
    for assetIdentifier, assetMetadata in assets.items():
        try:
            sourcePath = assetMetadata["source"]
            with open(sourcePath) as f:
                pass
        except FileNotFoundError:
            raise Exception(
                f"Missing asset.\nThe following asset referenced in the asset manifest could not be found:\n  {assetIdentifier}\n  {assetMetadata['source']}"
            )
        outputPath = os.path.join(outputDir + assetMetadata["slug"])

        if assetMetadata["type"] == "data":
            site["data"][assetMetadata["name"]] = json.load(
                open(assetMetadata["source"])
            )

        elif assetMetadata["type"] == "image":
            if assetMetadata["class"] in ["photograph", "art", "screen"]:
                ## if image is in HEIC format, force to save as JPEG, else preserve format
                # if outputPath.endswith(".heic"):
                #    outputPath = outputPath.replace(".heic", ".jpg")
                # if image already exists in outputDir, skip
                try:
                    with open(outputPath) as f:
                        pass
                # if image does not yet exist in outputDir, process and output
                except IOError:
                    if verbosity > 2:
                        print(f"{assetMetadata}: {outputPath}")
                    sourcePath = assetMetadata["source"]
                    os.makedirs(os.path.dirname(outputPath), exist_ok=True)
                    if "PANO" in outputPath:
                        compressedImage = processImage(sourcePath, 3000)
                    else:
                        compressedImage = processImage(sourcePath, 1200)
                    compressedImage["image"].save(outputPath)
                # thumbnailPath = os.path.join(
                #     outputDir + "thumbnails/" + assetMetadata["slug"]
                # )
                # try:
                #     with open(thumbnailPath) as f:
                #         pass
                # # repeat for creating thumbnails
                # except IOError:
                #     sourcePath = assetMetadata["source"]
                #     os.makedirs(os.path.dirname(thumbnailPath), exist_ok=True)
                #     thumbnailImage = processImage(sourcePath, 400)
                #     thumbnailImage["image"].save(thumbnailPath)
            # All other files in the manifest are (currently) copied straight through
            else:
                # if file already exists in outputDir, skip
                try:
                    with open(outputPath) as f:
                        pass
                # If file does not yet exist in outputDir, process and output
                except IOError:
                    source = assetMetadata["source"]
                    if verbosity > 2:
                        print(f"  {source} >> ", end="", flush=True)
                    os.makedirs(os.path.dirname(outputPath), exist_ok=True)
                    copyfile(source, outputPath)
                    if verbosity > 2:
                        print(outputPath)
        # Special handling of GPX files so that they don't go stale, needs reworking
        elif assetMetadata["type"] == "gpx":
            with open(assetMetadata["source"], "rb") as fileToHash:
                filedata = fileToHash.read()
            assets[assetIdentifier]["hash"] = md5(filedata).hexdigest()
            source = assetMetadata["source"]
            if verbosity > 2:
                print(f"  {source} >> ", end="", flush=True)
            os.makedirs(os.path.dirname(outputPath), exist_ok=True)
            copyfile(source, outputPath)
            if verbosity > 2:
                print(outputPath)
        else:
            # Copy all non 'image' assets straight through
            sourcePath = assetMetadata["source"]
            outputPath = os.path.join(outputDir + assetMetadata["slug"])
            if verbosity > 2:
                print(f"  {sourcePath} >> ", end="", flush=True)
            os.makedirs(os.path.dirname(outputPath), exist_ok=True)
            copyfile(sourcePath, outputPath)
            if verbosity > 2:
                print(outputPath)

    # Process image assets outside the manifest
    imageAssets = (f for f in Path(imagesDir).glob("**/*") if f.is_file())
    for image in imageAssets:
        if str(image).endswith((".jpg", ".png", ".webp")):
            slug = os.path.relpath(image, "assets/images")
            outputPath = os.path.join(outputDir + slug)
            # if image already exists in outputDir, skip
            try:
                with open(outputPath) as f:
                    pass
            # If image does not yet exist in outputDir, process and output
            except IOError:
                if verbosity > 2:
                    print(f"{image}: {outputPath}")
                os.makedirs(os.path.dirname(outputPath), exist_ok=True)
                compressedImage = processImage(image, 1000)
                compressedImage["image"].save(outputPath)
            # # repeat for creating thumbnails
            # thumbnailPath = os.path.join(outputDir + "thumbnails/" + slug)
            # try:
            #     with open(thumbnailPath) as f:
            #         pass
            # except IOError:
            #     os.makedirs(os.path.dirname(thumbnailPath), exist_ok=True)
            #     thumbnailImage = processImage(image, 400)
            #     thumbnailImage["image"].save(thumbnailPath)


def preprocessMetadata(filename, metadata):
    metadata["filename"] = filename
    if "uid" not in metadata:
        raise Exception(f"[ERROR] Document missing UID\n  {filename}")
    if "slug" not in metadata:
        raise Exception(f"[ERROR] Slug not set\n  {filename}")
    if "created" not in metadata:
        raise Exception(f"[ERROR] Document missing creation date\n  {filename}")
    if "updated" not in metadata:
        metadata["updated"] = metadata["created"]
    # Below is a temporary workaround while I transition to recording
    # creation/available dates as arrays rather than simple strings.
    # The previous form `created: 2023-09-15 19:55:46` will become the
    # following:
    # created:
    #   date: 2023-09-15 19:55:46
    #   location: Livno, Bosnia
    #   latitude: 44.30027181
    #   longitude: 15.85413135
    #   altitude: 605.1341
    if buildType != "drafts":
        if not isinstance(metadata["available"], datetime.date):
            metadata["location"] = metadata["available"].get(
                "location", metadata["created"]["location"]
            )
            metadata["available"] = metadata["available"]["date"]
            metadata["created"] = metadata["created"]["date"]
            if not isinstance(metadata["updated"], datetime.date):
                metadata["updated"] = metadata["available"]

    if metadata.get("source"):
        sourceCreator = metadata["source"].get("creator")
        sourceTitle = metadata["source"].get("title")
        # Date variable is reassigned if the more specific 'created' is available
        sourceDate = metadata["source"].get("year")
        sourceDate = metadata["source"].get("created", sourceDate)
        sourcePage = metadata["source"].get("page")
        sourceUrl = metadata["source"].get("url")
        if sourceUrl is not None:
            # closing parentheses are replaced with their encoded form so that URLs don't break the markdown/djot when they include parentheses
            sourceUrl = sourceUrl.replace(")", "%29")
        attrRich = ""
        attrPlain = ""
        if sourceCreator is not None:
            attrRich = attrPlain = sourceCreator
            if sourceTitle is not None and sourceUrl is not None:
                attrRich = f"{sourceCreator}, [{sourceTitle}]({sourceUrl})"
                attrPlain = f"{sourceCreator}, {sourceTitle}"
            elif sourceTitle is not None and sourceUrl is None:
                attrRich = attrPlain = f"{sourceCreator}, {sourceTitle}"
            elif sourceTitle is None and sourceUrl is not None:
                attrRich = f"[{sourceCreator}]({sourceUrl})"
        elif sourceTitle is not None:
            attrRich = attrPlain = sourceTitle
            if sourceTitle is not None and sourceUrl is not None:
                attrRich = f"[{sourceTitle}]({sourceUrl})"
        elif sourceTitle is None and sourceUrl is not None:
            attrRich = f"[link]({sourceUrl})"

        if sourcePage is not None:
            attrRich += f", Page {sourcePage}"
        if sourceDate is not None:
            if isinstance(sourceDate, datetime.date):
                sourceDate = sourceDate.year
            attrRich += f", {sourceDate}"

        # Add via link only after title has been altered
        if metadata.get("via") and metadata["via"].get("url"):
            viaUrl = metadata["via"]["url"]
            viaUrl = viaUrl.replace(")", "%29")
            attrRich += f" ([via]({viaUrl}))"

        attrRich = f"— {attrRich}"
        attrRich = run("jotdown", input=attrRich, text=True, capture_output=True).stdout
        metadata["attrRich"] = attrRich
        metadata["attrPlain"] = attrPlain

    if metadata.get("title") == None:
        try:
            metadata["title"] = attrPlain
        except UnboundLocalError:
            metadata["title"] = (
                metadata["created"].strftime("%B %e, %Y %-I.%M")
                + metadata["created"].strftime("%p").lower()
            )

    # ensure slug and title are stored as strings even if dates
    metadata["title"] = str(metadata["title"])
    metadata["slug"] = str(metadata["slug"])
    # create empty directories to be populated with 'interlinks' and 'backlinks'
    metadata["interlinks"] = []
    metadata["backlinks"] = []

    return metadata


def ingestDocuments():
    if verbosity > 1:
        print("Ingesting files ")
    fileList = getFiles()
    global documents
    global site
    documents = {}
    slugToTitleLookup = {}
    slugToUidLookup = {}
    siteSecondaries = []
    sitePrimaries = []
    siteSeries = []
    global tags
    tags = []
    global searchIndex
    searchIndex = []
    uuidCollisionLookup = []
    for filename in fileList:
        site["pagecount"] += 1
        if verbosity > 2:
            print(f"  {filename}")
        with open(filename) as f:
            rawMetadata, content = frontmatter.parse(f.read())
        metadata = preprocessMetadata(filename, rawMetadata)
        # insert wordcount into metadata and add to site wordcount
        metadata["wordcount"] = len(content.split())
        site["wordcount"] += metadata["wordcount"]
        if metadata["primary"] != "references":
            site["wordcountWithoutReferences"] += metadata["wordcount"]
        combined = metadata
        combined["content"] = content
        # Add each files content and metadata to global 'documents'
        documents[metadata["uid"]] = combined
        # Update global dictionary of internal links (part of backlinking strategy)
        slugToTitleLookup[metadata["slug"]] = metadata["title"]
        slugToUidLookup[metadata["slug"]] = metadata["uid"]
        # Append primaries and secondaries to globals
        sitePrimaries.append(metadata["primary"])
        siteSecondaries.append(metadata["secondary"])
        try:
            siteSeries.append(metadata["series"])
        except KeyError:
            pass
        try:
            for tag in metadata["tags"]:
                tags.append(tag)
        except KeyError:
            pass
        except TypeError:
            pass
        # Append to search index if not marked 'nofeed'
        options = metadata.get("options")
        if options is not None and "nofeed" in options:
            continue
        else:
            searchIndex.append(
                {
                    "title": metadata["title"],
                    "slug": metadata["slug"],
                    "content": content,
                    "primary": metadata["primary"],
                    "secondary": metadata["secondary"],
                }
            )
        uuidCollisionLookup.append(metadata["uid"])
        site["slugToUidLookup"] = slugToUidLookup
        site["slugToTitleLookup"] = slugToTitleLookup
    # Check for UUID collisions
    # More specifically, it checks for collisions using only the first 8
    # digits of each UUID, this is to ensure that this short prefix is enough to
    # uniquely identify each document and can be used widely for referencing.
    if verbosity > 2:
        print("Checking for collisions")
    prefixList = [key[0:8] for key in uuidCollisionLookup]
    if not len(set(prefixList)) == len(prefixList):
        sys.exit("CRITICAL ERROR: UUID prefix collision")
    # Dedupe global primaries/tags lists
    sitePrimaries = list(dict.fromkeys(sitePrimaries))
    siteSecondaries = list(dict.fromkeys(siteSecondaries))
    tags = list(dict.fromkeys(tags))
    # Insert count of site pages into site metadata
    site["primaries"] = sitePrimaries
    site["secondaries"] = siteSecondaries
    site["tags"] = tags
    # Transform search index into valid JSON
    searchIndex = json.dumps(searchIndex)
    if verbosity > 1:
        sitePagecount = site["pagecount"]
        print(f"  Ingested {sitePagecount} files")


def insertSubstitutions():
    if verbosity > 1:
        print("Performing substitutions")
    transclusionList = []
    REF_LINK_RE = re.compile(r"\[([^\]]*?)\]\((.*?::)([^)]+)\)")
    REF_SLUG_RE = re.compile(r"(?<!`)(slug::)([a-zA-Z0-9]*)")
    mergedData = {**documents, **assets}
    for key, page in documents.items():
        if verbosity > 2:
            print(f"  {key, page['title']}")
        text = page["content"]

        # Replace instances of site::pagecount and site::wordcount
        # accordingly. This is a bodge
        text = text.replace("site::pagecount", str(site["pagecount"]))
        text = text.replace("site::wordcount", "{:,}".format(site["wordcount"]))
        # Replace UUID document references in the source text
        # See "SEP: Reference documents by their URN/UUID" for the rationale
        # Find all valid UUID reference links, eg:
        #       [](slug::74534cff)
        #       [:Replace text](slug::74534cff)
        # REF_LINK_RE = re.compile(r"\[([^\]]*)\]\((![^)]+)\)")
        refSlugs = list(REF_SLUG_RE.findall(text))
        for refType, refShortId in refSlugs:
            match = f"{refType}{refShortId}"
            refId = ""
            for k in mergedData:
                if k.startswith(refShortId):
                    refId = k
            refSlug = f"/{mergedData[refId]['slug']}"
            replacement = refSlug
            text = text.replace(match, replacement)

        refLinks = list(REF_LINK_RE.findall(text))
        site["references"] += len(refLinks)
        # Process each match
        for refTextMatch, refType, refShortId in refLinks:
            match = f"[{refTextMatch}]({refType}{refShortId})"
            if refType not in [
                "link::",
                "in::",
                "sin::",
                "img::",
                "video::",
                "quote::",
            ]:
                raise Exception(
                    f"Unexpected Internal Reference type '{refType}' in document:\
                                {key}\n    match: {match}"
                )
            else:
                refId = ""
                for k in mergedData:
                    if k.startswith(refShortId):
                        refId = k
                # If refText is blank OR begins with a '::' lookup the title of
                # the linked document, else pass through.
                if refTextMatch.startswith("::") or refTextMatch == "":
                    try:
                        refText = mergedData[refId]["title"]
                    except:
                        refText = mergedData[refId]["description"]
                else:
                    refText = refTextMatch
                # Lookup slug of linked document
                refSlug = f"/{mergedData[refId]['slug']}"

                if refType in ["link::", "img::"]:
                    replacement = f"[{refText}]({refSlug})"
                    text = text.replace(match, replacement)

                elif refType in ["quote::"]:
                    refSrc = mergedData[refId]["source"]["creator"]
                    refText = mergedData[refId]["content"].replace("\n\n", "\n>\n> ")
                    replacement = f"> {refText}\\\n> — [{refSrc}]({refSlug})"
                    text = text.replace(match, replacement)

                elif refType in ["in::", "sin::"]:
                    transcludePayload = {
                        "refType": refType,
                        "insrc_uuid": key,
                        "insrc_match": match,
                        "refId": refId,
                        "inref_linktext": refText,
                        "inref_linktarget": refSlug,
                    }
                    transclusionList.append(transcludePayload)

                elif refType == "video::":
                    # Assemble payload
                    replacement = f'``` =html\n<video autoplay muted controls style="display: block;margin: auto;width: 60%;">\n<source src="{refSlug}" type="video/mp4">\nYour browser does not support the video tag.\n</video>\n```'
                    # Find and replace the original link with the updated one
                    text = text.replace(match, replacement)
            # If no matching UUID found for reference, raise Exception
            if refId == "":
                raise Exception(
                    f"\nUnmatched UUID reference:\n"
                    f"  document: {key}\n"
                    f"  {refShortId} does not reference an existing document"
                )

        # Write modified plaintext back to document variable once all
        # substitutions have been carried out.
        page["content"] = text

        # Carry out transclusions (these must occur after all other link references to ensure such links are present in transcluded elements)
        for transclude in transclusionList:
            insrc_uuid = transclude["insrc_uuid"]
            insrc_match = transclude["insrc_match"]
            refId = transclude["refId"]
            inref_transclude = run(
                "jotdown",
                input=documents[refId]["content"],
                text=True,
                capture_output=True,
            ).stdout
            refText = transclude["inref_linktext"]
            refSlug = transclude["inref_linktarget"]
            # Assemble payload
            if transclude["refType"] == "in::":
                replacement = f'``` =html\n<figure class="transclusion">\n<p>{inref_transclude}</p>\n<figcaption> <em>from</em> <a href="{refSlug}">{refText}</a></figcaption>\n</figure>\n```'
            else:
                replacement = f'``` =html\n<figure class="silent-transclusion">\n{inref_transclude}\n</figure>\n```'
            # Find and replace the original link with the assembled transclusion
            documents[insrc_uuid]["content"] = documents[insrc_uuid]["content"].replace(
                insrc_match, replacement
            )


def generateHTML():
    if verbosity > 1:
        print("Generating HTML")
    for key, page in documents.items():
        # Convert lightweight markup to HTML
        parsed = run("jotdown", input=page["content"], text=True, capture_output=True)
        html = parsed.stdout
        documents[key]["html"] = html


def buildBacklinks():
    # `interlinks` is a metadata element listing the UIDs of all the documents
    # that the document points OUT to.
    # `backlinks` is a metadata element listing the UIDs of the documents that
    # point IN TO the document.
    if verbosity > 1:
        print("Building backlinks ")
    global interlinkCount
    interlinkCount = 0

    INLINE_LINK_RE = re.compile(
        r"\[[^\]]*(?:\[[^\]]*\][^\]]*)*\]\(\/([^)]*)\)", re.DOTALL
    )
    FOOTNOTE_LINK_URL_RE = re.compile(r"\[.+?\]:\s\/(.*)", re.DOTALL)
    for key, page in documents.items():
        if "nobacklinks" in page.get("options", ""):
            continue
        else:
            if verbosity > 2:
                print(page["title"])
            text = page["content"]
            interlinks = []

            # locate all internal references between documents.
            inlineRefs = list(INLINE_LINK_RE.findall(text))
            footnoteRefs = list(FOOTNOTE_LINK_URL_RE.findall(text))
            combinedRefs = inlineRefs + footnoteRefs

            # For each internal link, use its slug to lookup the UUID of the
            # document it points too, add that to a list called 'interlinks'
            for slug in combinedRefs:
                try:
                    link_uid = site["slugToUidLookup"][slug]
                    interlinks.append(link_uid)
                    interlinkCount += 1
                except KeyError:
                    # Ignore errors from slugs that: point to feeds and images,
                    # start with '$', or appear in the exlude list
                    if (
                        slug.startswith(("feeds/", "images/", "$"))
                        or slug.endswith(
                            (
                                ".jpg",
                                ".webp",
                                ".png",
                                ".svg",
                                ".pdf",
                                ".gif",
                            )
                        )
                        or slug in ["publickey", "humans.txt", "build.py"]
                    ):
                        continue
                    # If none of the above catch the error, print the title,
                    # uuid, and slug to stdout but continue with build.
                    else:
                        print()
                        print(page["title"], key)
                        print(f"KeyError: {slug}")
                        continue

            # Deduplicate interlinks and store
            interlinks = list(dict.fromkeys(interlinks))
            documents[key]["interlinks"] = interlinks
            # Insert document key into backlinks of referenced documents
            for interlink_key in interlinks:
                documents[interlink_key]["backlinks"].append(key)

            site["backlinks"] += len(interlinks)


def buildCollections():
    global collections
    collections = {}
    global sitemap
    sitemap = []
    ## Need to build in a check to make sure there are no collisions between primaries/collections/tags
    for primary in site["primaries"]:
        collections.update({primary: []})
    for secondary in site["secondaries"]:
        collections.update({secondary: []})
    # Create empty collection for each tag
    for tag in tags:
        collections.update({tag: []})
    # Create empty 'everything' collection
    collections.update({"everything": []})
    # Add each page to collections for each of its tags unless 'nofeed' is found
    # in metadata['options'], (though even those will appear in the sitemap variable)
    # Only 'drafts' are omitted from the sitemap
    for key, page in sorted(
        documents.items(), key=lambda k_v: k_v[1]["available"], reverse=True
    ):
        options = page.get("options")
        if options is not None and "nofeed" in options:
            sitemap.append(page)
            continue
        elif page["slug"].startswith("drafts"):
            continue
        else:
            sitemap.append(page)
            collections["everything"].append(page)
            collections[page["primary"]].append(page)
            collections[page["secondary"]].append(page)
            try:
                for tag in page["tags"]:
                    collections[tag].append(page)
            except KeyError:
                pass
            except TypeError:
                pass


def outputHTML():
    if verbosity > 1:
        print("Generating Hypertext")
    for key, page in documents.items():
        collection = []
        if verbosity > 2:
            print(f"  {page['filename']} >> ", end="", flush=True)
        templateFile = page.get("layout", defaultTemplate)
        template = env.get_template(templateFile)
        try:
            for include in page["collection"]["include"]:
                collection.append(collections[include])
            # If a page has multiple tags in its collection, sort chronologicaly
            if len(page["collection"]["include"]) > 0:
                collection = [item for sublist in collection for item in sublist]
                collection.sort(key=lambda x: x["available"], reverse=True)
        except KeyError:
            pass
        # Generate the static page
        output = template.render(
            documents=documents,
            assets=assets,
            collections=collections,
            collection=collection,
            page=page,
            site=site,
            searchIndex=searchIndex,
        )
        # Ready output path
        outputPath = f"{outputDir + page['slug']}/index.html"
        # Create output directory tree
        os.makedirs(os.path.dirname(outputPath), exist_ok=True)
        # Write final html
        with open(outputPath, "w") as f:
            f.write(output)
        if verbosity > 2:
            print(f"{outputPath}")


def renderFeed(feedName):
    slug = os.path.join("feeds", feedName)
    feedPath = os.path.join(outputDir, slug, "index.xml")
    template = env.get_template(feedTemplate)
    feedContent = template.render(
        site=site,
        slug=slug,
        collection=feedName,
        feed=collections[feedName],
    )

    feed = {"name": feedName, "output": feedContent, "path": feedPath}
    return feed


def outputFeeds():
    if verbosity > 1:
        print("Generating Feeds")

    feedList = site["primaries"] + site["secondaries"] + ["everything"]

    for entry in feedList:
        if verbosity > 2:
            print(f"  {entry} >> ", end="", flush=True)

        feed = renderFeed(entry)
        os.makedirs(os.path.dirname(feed["path"]), exist_ok=True)
        with open(feed["path"], "w") as f:
            f.write(feed["output"])

        if verbosity > 2:
            print(feed["path"])

    # Render feed stylesheet (XSL)
    if verbosity > 1:
        print("Creating XLS Stylesheet for Feeds")

    template = env.get_template(feedXlsTemplate)
    outputPath = os.path.join(outputDir, "feed.xsl")
    os.makedirs(os.path.dirname(outputPath), exist_ok=True)
    output = template.render(site=site)
    with open(outputPath, "w") as f:
        f.write(output)

    if verbosity > 2:
        print(f"{outputPath}")


def outputSitemap():
    if verbosity > 1:
        print("Generating Sitemap")
    template = env.get_template(sitemapTemplate)
    output = template.render(sitemap=sitemap, site=site)
    outputPath = outputDir + "sitemap.xml"
    with open(outputPath, "w") as f:
        f.write(output)
    if verbosity > 2:
        print(f"  {outputPath}")


def copyAssets():
    # Copy assets (excluding images, including SVGs)
    if verbosity > 1:
        print("Copying Assets")
    assets = (f for f in Path(assetsDir).glob("**/*") if f.is_file())
    for item in assets:
        # Copy SVGs, GIFs, and MP4s to output dir.
        if str(item).endswith((".svg", ".mp4", ".gif")):
            if verbosity > 2:
                print(f"  {item} >> ", end="", flush=True)
            outputPath = outputDir + ("/".join(Path(item).parts[1:]))
            os.makedirs(os.path.dirname(outputPath), exist_ok=True)
            copyfile(item, outputPath)
            if verbosity > 2:
                print(outputPath)
        elif "/templates/" not in str(item) and "/images/" not in str(item):
            if verbosity > 2:
                print(f"  {item} >> ", end="", flush=True)
            outputPath = outputDir + ("/".join(Path(item).parts[1:]))
            # Create output directory
            os.makedirs(os.path.dirname(outputPath), exist_ok=True)
            copyfile(item, outputPath)
            if verbosity > 2:
                print(outputPath)


profend("Imports, setup, and definitions")


# Run
if buildDepth in ["normal", "production"]:
    profbegin()
    processAssets()
    profend("Processed asset manifest")
    profbegin()
    ingestDocuments()
    profend("\033[1mIngested documents\033[0m")
    profbegin()
    insertSubstitutions()
    refCount = site["references"]
    profend(f"Inserted {refCount} substitutes")
    profbegin()
    generateHTML()
    profend("\033[1mGenerated HTML\033[0m")
    profbegin()
    buildBacklinks()
    profend(f"Built {interlinkCount} backlinks")
    profbegin()
    buildCollections()
    profend("Built collections")
    profbegin()
    outputHTML()
    profend("Wrote out HTML files")
    profbegin()
    outputFeeds()
    profend("Built feeds")
    profbegin()
    outputSitemap()
    profend("Built sitemap")
    profbegin()
    copyAssets()
    profend("Copied assets")
else:
    profbegin()
    ingestDocuments()
    profend("\033[1mIngested documents\033[0m")
    profbegin()
    buildCollections()
    profend("Built collections")
    profbegin()
    generateHTML()
    profend("\033[1mGenerated HTML\033[0m")
    profbegin()
    outputHTML()
    profend("Wrote out HTML files")

# Profiling
# Change back to directory that user was in prior to build
os.chdir(startDirectory)

end = time.time()
duration = round((end - start), 3)
buildtime = time.strftime("%H:%M:%S", time.localtime(end))
pagecount = site["pagecount"]

if verbosity > 0:
    print(f"{buildtime} Built {pagecount} in {duration} seconds")

#