forked from JesperDramsch/python-deadlines
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathimport_python_official.py
More file actions
495 lines (409 loc) · 18 KB
/
Copy pathimport_python_official.py
File metadata and controls
495 lines (409 loc) · 18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
# Standard library
import re
from datetime import datetime
from datetime import timedelta
from datetime import timezone
from pathlib import Path
# Third-party
import pandas as pd
import requests
from icalendar import Calendar
# Local imports
try:
from logging_config import get_tqdm_logger
from tidy_conf import fuzzy_match
from tidy_conf import load_conferences
from tidy_conf import merge_conferences
from tidy_conf.date import create_nice_date
from tidy_conf.deduplicate import deduplicate
from tidy_conf.titles import tidy_df_names
from tidy_conf.utils import fill_missing_required
from tidy_conf.yaml import load_title_mappings
from tidy_conf.yaml import write_df_yaml
except ImportError:
from .logging_config import get_tqdm_logger
from .tidy_conf import fuzzy_match
from .tidy_conf import load_conferences
from .tidy_conf import merge_conferences
from .tidy_conf.date import create_nice_date
from .tidy_conf.deduplicate import deduplicate
from .tidy_conf.titles import tidy_df_names
from .tidy_conf.utils import fill_missing_required
from .tidy_conf.yaml import load_title_mappings
from .tidy_conf.yaml import write_df_yaml
logger = get_tqdm_logger(__name__)
def fill_links_from_history(df_ics: pd.DataFrame, df_yml: pd.DataFrame) -> pd.DataFrame:
"""Fill missing links in ICS data from historical conference data.
For conferences without links, look up the conference name in historical data
and use that link, replacing any year references with the current year.
Parameters
----------
df_ics : pd.DataFrame
DataFrame with ICS conference data (may have empty links)
df_yml : pd.DataFrame
DataFrame with existing conference data from YAML files
Returns
-------
pd.DataFrame
DataFrame with missing links filled where historical data exists
"""
if df_yml.empty:
return df_ics
# Create a lookup of conference names to their most recent links
# Group by normalized conference name and get the most recent entry
historical_links = {}
for _, row in df_yml.iterrows():
conf_name = row.get("conference", "")
link = row.get("link", "")
year = row.get("year", 0)
# Keep the most recent link for each conference
if conf_name and link and (conf_name not in historical_links or year > historical_links[conf_name][1]):
historical_links[conf_name] = (link, year)
filled_count = 0
for idx, row in df_ics.iterrows():
link = row.get("link", "")
if not link or len(str(link).strip()) == 0:
conf_name = row.get("conference", "")
target_year = row.get("year", datetime.now(tz=timezone.utc).year)
if conf_name in historical_links:
hist_link, hist_year = historical_links[conf_name]
# Replace the historical year with the target year in the link
new_link = re.sub(
rf"\b{hist_year}\b",
str(target_year),
str(hist_link),
)
df_ics.at[idx, "link"] = new_link
filled_count += 1
logger.debug(
f"Filled link for '{conf_name}' from historical data: {new_link}",
)
if filled_count > 0:
logger.info(f"Filled {filled_count} missing links from historical conference data")
return df_ics
def ics_to_dataframe() -> pd.DataFrame:
"""Parse an .ics file and return a DataFrame with the event data.
Returns
-------
pd.DataFrame: DataFrame containing parsed conference events
Raises
------
ConnectionError: If unable to fetch the calendar data
ValueError: If calendar data is invalid
"""
calendar_url = (
"https://www.google.com/calendar/ical/[email protected]/public/basic.ics"
)
# Validate URL scheme for security
if not calendar_url.startswith("https://"):
raise ValueError("Only HTTPS URLs are allowed for security")
logger.info(f"Fetching calendar data from: {calendar_url}")
try:
response = requests.get(calendar_url, timeout=30)
response.raise_for_status()
calendar_data = response.content
if not calendar_data:
raise ValueError("Empty calendar data received")
calendar = Calendar.from_ical(calendar_data)
logger.info(f"Successfully parsed calendar data ({len(calendar_data)} bytes)")
except requests.exceptions.RequestException as e:
logger.error(f"Failed to fetch calendar data: {e}")
raise ConnectionError(
f"Unable to fetch calendar from {calendar_url}: {e}",
) from e
except Exception as e:
logger.error(f"Failed to parse calendar data: {e}")
raise ValueError(f"Invalid calendar data: {e}") from e
link_desc = re.compile(
r".*<a .*?href=\"? ?((?:https|http):\/\/[\w\.\/\-\?= ]+)\"?.*?>(.*?)[#0-9 ]*<\/?a>.*",
)
# Initialize a list to hold event data
event_data = []
processed_events = 0
skipped_events = 0
# Iterate over each event in the Calendar
for component in calendar.walk():
if component.name == "VEVENT":
try:
# Extract event details with error checking
conference = str(component.get("summary", "Unknown Conference"))
# Safely extract dates
dtstart = component.get("dtstart")
dtend = component.get("dtend")
if not dtstart or not dtend:
logger.warning(
f"Skipping event '{conference}' - missing date information",
)
skipped_events += 1
continue
start = dtstart.dt
end = dtend.dt - timedelta(days=1)
# If the event is all day, the date might be of type 'date' (instead of 'datetime')
# Adjust format accordingly
start = start.strftime("%Y-%m-%d")
end = end.strftime("%Y-%m-%d")
year = int(start[:4])
except (AttributeError, ValueError, TypeError) as e:
logger.warning(f"Skipping event due to date parsing error: {e}")
skipped_events += 1
continue
# Process description and extract links
try:
raw_description = str(component.get("description", ""))
if not raw_description:
logger.warning(
f"Event '{conference}' has no description, skipping link extraction",
)
link = ""
else:
# Clean HTML entities and format description
description = re.sub(
r"(?:\\s| |\\|\'|<br />|<br>|</[^a][^>]*>|<[^a/][^>]*>)+",
" ",
"<a "
+ "<a ".join(
raw_description.replace("\n", "")
.replace(
""", '"')
.replace(""",
'"',
)
.replace("&", "&")
.replace(""", '"')
.replace("'", "'")
.replace("<", "<")
.replace(">", ">")
.split("<a ")[1:],
),
)
# Extract link and conference name from description
m = re.match(link_desc, description)
if m:
link = m.group(1).strip()
conference2 = m.group(2).strip()
if conference2:
conference = conference2
else:
logger.debug(f"No link found in description for '{conference}'")
link = ""
except Exception as e:
logger.warning(f"Error processing description for '{conference}': {e}")
link = ""
location = str(component.get("location", ""))
# Append this event's details to the list
event_data.append([conference, year, "TBA", start, end, link, location])
processed_events += 1
# Log processing summary
logger.info(
f"Calendar processing complete: {processed_events} events processed, {skipped_events} skipped",
)
# Convert the list into a pandas DataFrame
df = pd.DataFrame(
event_data,
columns=["conference", "year", "cfp", "start", "end", "link", "place"],
)
if df.empty:
logger.warning("No events were successfully processed from calendar")
return df
# Strip whitespace from applicable columns
try:
df_obj = df.select_dtypes("object")
df[df_obj.columns] = df_obj.apply(lambda x: x.str.strip())
logger.info(f"Created DataFrame with {len(df)} conference entries")
except Exception as e:
logger.error(f"Error cleaning DataFrame: {e}")
return df
def main(year=None, base="") -> bool:
"""Import Python conferences from a Google Calendar .ics file.
Args:
year: Target year for filtering (defaults to current year)
base: Base path for file operations
Returns
-------
bool: True if import was successful, False otherwise
"""
logger.info("Starting Python Official calendar import")
# If no year is provided, use the current year
if year is None:
year = datetime.now(tz=timezone.utc).year
logger.info(f"Importing conferences for year: {year}")
try:
# Create the necessary files if they don't exist
data_path = Path(base, "_data")
tmp_path = Path(base, ".tmp")
tmp_path.mkdir(exist_ok=True, parents=True)
data_path.mkdir(exist_ok=True, parents=True)
target_file = Path(data_path, "conferences.yml")
cache_file = Path(tmp_path, ".conferences_ics.csv")
logger.info(f"Using data path: {data_path}")
logger.info(f"Using cache file: {cache_file}")
# Load the existing conference data
logger.info("Loading existing conference data")
df_yml = load_conferences()
df_new = pd.DataFrame(columns=df_yml.columns)
# Parse your .ics file and only use future events in the current year
logger.info("Parsing ICS calendar data")
df_ics = ics_to_dataframe()
if df_ics.empty:
logger.warning("No conference data retrieved from calendar")
return False
# Try to fill missing links from historical conference data
logger.info("Filling missing links from historical data")
df_ics = fill_links_from_history(df_ics, df_yml)
# Filter out entries with empty or missing links
initial_count = len(df_ics)
df_ics = df_ics[df_ics["link"].str.len() > 0]
filtered_count = initial_count - len(df_ics)
if filtered_count > 0:
logger.info(f"Filtered out {filtered_count} entries without valid links")
if df_ics.empty:
logger.warning("No conferences with valid links after filtering")
return False
except Exception as e:
logger.error(f"Failed to initialize import process: {e}")
return False
# Load old ics dataframe from cached data
try:
# Load the old ics dataframe from cache
logger.info("Loading cached ICS data")
df_ics_old = pd.read_csv(cache_file, na_values=None, keep_default_na=False)
logger.info(f"Loaded {len(df_ics_old)} cached entries")
except FileNotFoundError:
logger.info("No cache file found, starting fresh")
df_ics_old = pd.DataFrame(columns=df_ics.columns)
except Exception as e:
logger.error(f"Error loading cache file: {e}")
df_ics_old = pd.DataFrame(columns=df_ics.columns)
try:
# Load and apply the title mappings, remove years from conference names
logger.info("Applying title mappings and cleaning data")
df_ics = tidy_df_names(df_ics)
# Store the new ics dataframe to cache
df_cache = df_ics.copy()
# Get the difference between the old and new dataframes
df_diff = pd.concat([df_ics_old, df_ics]).drop_duplicates(keep=False)
# Deduplicate the new dataframe
# CRITICAL: Must group by both conference AND year to avoid losing multi-year entries
df_ics = deduplicate(df_diff, ["conference", "year"])
if df_ics.empty:
logger.info("No new conferences found in official Python source.")
return True # Not an error, just no new data
except Exception as e:
logger.error(f"Error processing conference data: {e}")
return False
try:
_, reverse_titles = load_title_mappings(reverse=False)
# Fuzzy match the new data with the existing data
logger.info(f"Starting fuzzy matching for years {year} to {year + 9}")
processed_years = 0
for y in range(year, year + 10):
# Skip years that are not in the new data
if df_ics.loc[df_ics["year"] == y].empty or df_yml[df_yml["year"] == y].empty:
# Concatenate the new data with the existing data
df_new = pd.concat(
[
df_new,
df_yml[df_yml["year"] == y],
df_ics.loc[df_ics["year"] == y],
],
ignore_index=True,
)
continue
df_merged, df_remote, merge_report = fuzzy_match(
df_yml[df_yml["year"] == y],
df_ics.loc[df_ics["year"] == y],
)
logger.info(
f"Merge report: {merge_report.exact_matches} exact, "
f"{merge_report.fuzzy_matches} fuzzy, {merge_report.no_matches} no match",
)
df_merged["year"] = y
diff_idx = df_merged.index.difference(df_remote.index)
df_missing = df_merged.loc[diff_idx, :].sort_values("start")
df_merged = df_merged.drop(["conference"], axis=1)
df_merged = deduplicate(df_merged)
df_remote = deduplicate(df_remote)
df_merged = merge_conferences(df_merged, df_remote)
# Concatenate the new data with the existing data
df_new = pd.concat([df_new, df_merged], ignore_index=True)
for _index, row in df_missing.iterrows():
reverse_title_data = reverse_titles.get(row["conference"])
if reverse_title_data is None:
reverse_title = f"{row['conference']} {row['year']}"
else:
# Get the first variation from the reverse title data
reverse_title_data = reverse_title_data.get("variations")
if reverse_title_data:
reverse_title = f"{reverse_title_data[0]} {row['year']}"
else:
reverse_title = f"{row['conference']} {row['year']}"
timezone_str = row["timezone"] if isinstance(row["timezone"], str) else "UTC"
dates = f'{create_nice_date(row)["date"]} ({timezone_str})'
link = f'<a href="{row["link"]}">{row["conference"]}</a>'
out = f""" * name of the event: {reverse_title}
* type of event: conference
* focus on Python: yes
* approximate number of attendees: Unknown
* location (incl. country): {row["place"]}
* dates/times/recurrence (incl. time zone): {dates})
* HTML link using the format <a href="http://url/">name of the event</a>: {link}"""
with Path("missing_conferences.txt").open("a") as f:
f.write(out + "\n\n")
Path(".tmp").mkdir(exist_ok=True, parents=True)
Path(".tmp", f"{reverse_title}.ics".lower().replace(" ", "-")).write_text(
f"""BEGIN:VCALENDAR
VERSION:2.0
BEGIN:VEVENT
SUMMARY:{reverse_title}
DTSTART;VALUE=DATE:{row["start"].strftime("%Y%m%d")}
DTEND;VALUE=DATE:{row["end"].strftime("%Y%m%d")}
DESCRIPTION:<a href="{row.link}">{ reverse_title }</a>
LOCATION:{ row.place }
END:VEVENT
END:VCALENDAR""",
)
processed_years += 1
logger.info(f"Fuzzy matching complete: processed {processed_years} years")
# Fill in missing required fields
logger.info("Filling missing required fields")
df_new = fill_missing_required(df_new)
# Write the new data to the YAML file
logger.info(f"Writing {len(df_new)} conference entries to {target_file}")
write_df_yaml(df_new, target_file)
# Save the new dataframe to cache
logger.info(f"Saving cache to {cache_file}")
df_cache.to_csv(cache_file, index=False)
logger.info("Python Official calendar import completed successfully")
return True
except Exception as e:
logger.error(f"Error during fuzzy matching and data processing: {e}")
return False
if __name__ == "__main__":
import argparse
import sys
parser = argparse.ArgumentParser(
description="Import Python conferences from official calendar",
)
parser.add_argument(
"--year",
type=int,
help="Year to import (defaults to current year)",
)
parser.add_argument("--base", type=str, default="", help="Base path for data files")
parser.add_argument(
"--log-level",
type=str,
default="INFO",
choices=["DEBUG", "INFO", "WARNING", "ERROR"],
help="Set logging level",
)
args = parser.parse_args()
# Set up logging
from logging_config import setup_logging
setup_logging(level=args.log_level)
# Run the import
success = main(year=args.year, base=args.base)
if not success:
logger.error("Import failed")
sys.exit(1)
logger.info("Import completed successfully")