website/parse_openai.py at master · bwendell/website · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
import re
import sys
from html.parser import HTMLParser

class OpenAIParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.in_article = False
        self.output = []
        self.current_tag = ""
        self.href = ""
        self.in_title = False
        self.title = ""
        self.in_h1 = False
        self.extracted_title = ""
        self.capture_text = False
        self.list_depth = 0

    def handle_starttag(self, tag, attrs):
        attrs_dict = dict(attrs)
        classes = attrs_dict.get('class', '')

        if tag == 'article':
            self.in_article = True

        if self.in_article:
            if tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
                self.current_tag = tag
                self.output.append(f"\n\n{('#' * int(tag[1]))} ")
                if tag == 'h1':
                    self.in_h1 = True
            elif tag == 'p':
                self.current_tag = 'p'
                self.output.append("\n\n")
            elif tag == 'li':
                self.output.append("\n- ")
            elif tag == 'a':
                self.href = attrs_dict.get('href', '')
                self.output.append("[")
            elif tag == 'strong' or tag == 'b':
                self.output.append("**")
            elif tag == 'em' or tag == 'i':
                self.output.append("*")
            elif tag == 'u':
                pass # Skip underline formatting if strictly desired, or implement
            elif tag == 'code':
                self.output.append("`")

    def handle_endtag(self, tag):
        if tag == 'article':
            self.in_article = False

        if self.in_article:
            if tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
                if tag == 'h1':
                    self.in_h1 = False
            elif tag == 'a':
                self.output.append(f"]({self.href})")
                self.href = ""
            elif tag == 'strong' or tag == 'b':
                self.output.append("**")
            elif tag == 'em' or tag == 'i':
                self.output.append("*")
            elif tag == 'code':
                self.output.append("`")

    def handle_data(self, data):
        if self.in_article:
            content = data.replace('\n', ' ').strip()
            if content:
                # Special handling for H1 to capture it if needed, though we get it from metadata too
                if self.in_h1:
                    self.extracted_title = content

                # Append text with a space if needed to prevent concatenation
                # self.output.append(content + " ")
                # Better: clean text
                self.output.append(data.replace('\n', ' '))


# Simple regex extraction for metadata because HTMLParser is stream based and <head> is before <article>
with open('openai_post.html', 'r', encoding='utf-8') as f:
    html_content = f.read()

# Extract Title
title_match = re.search(r'<title>(.*?)</title>', html_content)
title = title_match.group(1).split('|')[0].strip() if title_match else "Scaling PostgreSQL"

# Extract Date
date_match = re.search(r'January \d{1,2}, \d{4}', html_content)
date = date_match.group(0) if date_match else "2026-01-22"

# Convert Date to YYYY-MM-DD
try:
    from datetime import datetime
    dt = datetime.strptime(date, '%B %d, %Y')
    date_iso = dt.strftime('%Y-%m-%d')
except:
    date_iso = date

# Extract Summary
desc_match = re.search(r'<meta name="description" content="(.*?)"', html_content)
summary = desc_match.group(1) if desc_match else ""

# Extract Content
parser = OpenAIParser()
parser.feed(html_content)
content_md = "".join(parser.output)

# Clean up Markdown
# Remove excessive newlines
content_md = re.sub(r'\n{3,}', '\n\n', content_md)
# Fix link spacing [ text ]( url ) -> [text](url)
content_md = re.sub(r'\[\s+', '[', content_md)
content_md = re.sub(r'\s+\]', ']', content_md)
# Fix bold spacing
content_md = re.sub(r'\*\*\s+', '**', content_md)
content_md = re.sub(r'\s+\*\*', '**', content_md)

# Construct Final MDX
mdx_content = f"""---
title: "{title}"
date: "{date_iso}"
type: "blog"
original_url: "https://openai.com/index/scaling-postgresql/"
tags: ["postgresql", "scaling", "database", "distributed-systems"]
summary: "{summary}"
---

{content_md.strip()}
"""

with open('content/openai-postgres.mdx', 'w', encoding='utf-8') as f:
    f.write(mdx_content)

print(f"Generated content/openai-postgres.mdx with title: {title}")