Updates

2026-03-03 05:36:56 +01:00
parent d1ae498a2f
commit 57f1ecf5c3
7 changed files with 217 additions and 20 deletions
@@ -0,0 +1,159 @@
+// @ts-ignore
+const articles_objects = [];
+
+const substack_href = "https://substack.com/@benjamintoby";
+
+// @ts-ignore
+async function sleep(wait) {
+    return new Promise((res) => {
+        setTimeout(() => {
+            res(true);
+        }, wait);
+    });
+}
+
+function grabContentHeight() {
+    return document.querySelector(
+        "div[style='max-width: 568px;']",
+        // @ts-ignore
+    )?.offsetHeight;
+}
+
+async function scrollToEnd() {
+    let last_content_height = grabContentHeight();
+
+    while (true) {
+        window.scrollTo({
+            top: document.body.scrollHeight,
+            behavior: "smooth",
+        });
+
+        await sleep(5000);
+
+        const current_content_height = grabContentHeight();
+
+        if (current_content_height > last_content_height) {
+            last_content_height = current_content_height;
+        } else {
+            break;
+        }
+    }
+}
+
+async function main() {
+    await scrollToEnd();
+
+    const articles = Array.from(
+        document.querySelectorAll("div[role='article']"),
+    );
+
+    console.log(`Handling ${articles.length} Articles ...`);
+
+    for (let i = 0; i < articles.length; i++) {
+        let present_articles = Array.from(
+            document.querySelectorAll("div[role='article']"),
+        );
+
+        console.log(`Found ${present_articles.length} Present Articles!`);
+
+        while (i > present_articles.length - 1) {
+            window.scrollTo({
+                top: document.body.scrollHeight,
+                behavior: "smooth",
+            });
+
+            console.log(`Searching for Article #${i} ...`);
+
+            await sleep(5000);
+
+            present_articles = Array.from(
+                document.querySelectorAll("div[role='article']"),
+            );
+        }
+
+        const article = present_articles[i];
+
+        console.log(`Handling Article #${i} ...`);
+
+        const content_div = article.querySelector(`.FeedProseMirror`);
+        const date_link = Array.from(article.querySelectorAll("a")).find((a) =>
+            Boolean(a.getAttribute("title")),
+        );
+
+        if (!content_div) continue;
+
+        const content_div_first_paragraph = content_div.querySelector(`p`);
+
+        if (!content_div_first_paragraph) continue;
+
+        let window_url = window.location.href;
+        const initial_text_content = content_div.textContent;
+
+        const article_object = {
+            title: initial_text_content,
+            content: initial_text_content,
+            html: content_div.innerHTML,
+            images: [],
+            date: date_link?.getAttribute("title"),
+        };
+
+        const article_images = Array.from(
+            article.querySelectorAll("picture img"),
+        );
+
+        if (article_images?.[0]) {
+            for (let img = 0; img < article_images.length; img++) {
+                if (img > 0) {
+                    const article_image = article_images[img];
+                    // @ts-ignore
+                    const article_image_srcset = article_image.srcset;
+                    const largest_image = article_image_srcset
+                        .split(` `)
+                        .at(-2);
+                    // @ts-ignore
+                    article_object.images.push(largest_image);
+                }
+            }
+        }
+
+        const more_content = Array.from(article.querySelectorAll("a"))
+            .find((el) => el.textContent.includes("See more"))
+            ?.click();
+
+        await sleep(2000);
+
+        let new_window_url = window.location.href;
+
+        if (new_window_url === window_url) {
+            const new_article_content =
+                article.querySelector(`.FeedProseMirror`);
+
+            if (new_article_content) {
+                article_object.content = new_article_content.textContent;
+                article_object.html = new_article_content.innerHTML;
+            }
+
+            articles_objects.push(article_object);
+        } else {
+            const text_sample = content_div_first_paragraph.textContent;
+            const target_content_div = Array.from(
+                document.querySelectorAll(".ProseMirror.FeedProseMirror"),
+            ).find((el) => el.textContent.includes(text_sample));
+
+            if (target_content_div) {
+                article_object.content = target_content_div.textContent;
+                article_object.html = target_content_div.innerHTML;
+            }
+
+            articles_objects.push(article_object);
+
+            window.history.back();
+            await sleep(2000);
+        }
+    }
+
+    // @ts-ignore
+    console.log(articles_objects);
+}
+
+main();