+
+
+
+
+
+
+
+
+
+
diff --git a/talp_report/talp_time_series.py b/talp_report/talp_time_series.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d1fccd04b14a3ec9fde51c1b17c5d4d4ba5f2a6
--- /dev/null
+++ b/talp_report/talp_time_series.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import pandas as pd
+import os
+from jinja2 import Environment, FileSystemLoader
+import sqlite3
+from datetime import datetime
+
+from talp_variables import TALP_DB_COLUMN_METADATA, TALP_DB_COLUMN_TALP_OUPUT, TALP_DB_COLUMN_TIMESTAMP, TALP_DEFAULT_REGION_NAME, TALP_TABLE_NAME, TALP_TEMPLATE_PATH
+
+
+def date_time_to_string(datetime):
+ return datetime.strftime("%d.%m.%Y %H:%M")
+
+
+def render_template(directory, template_name, **context):
+ # Set up Jinja2 environment and load the template
+ env = Environment(loader=FileSystemLoader(directory))
+ template = env.get_template(template_name)
+
+ # Render the template with the provided context
+ return template.render(context)
+
+
+def extract_region_names_from_df(df):
+ region_names = set()
+ talp_outputs = df[TALP_DB_COLUMN_TALP_OUPUT].tolist()
+ for talp_output in talp_outputs:
+ raw_data = json.loads(talp_output)
+ for entry in raw_data['popMetrics']:
+ region_names.add(entry['name'])
+ return list(region_names)
+
+
+def get_formatted_timestamps(df):
+ timestamps_df = df[TALP_DB_COLUMN_TIMESTAMP].tolist()
+ timestamps = []
+ for timestamp in timestamps_df:
+ parsed_date = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")
+ formatted_date = date_time_to_string(parsed_date)
+ timestamps.append(formatted_date)
+ return timestamps
+
+
+def extract_metadata_from_df(df):
+ metadata_obj = {}
+ timestamps = get_formatted_timestamps(df)
+ metadatas = df[TALP_DB_COLUMN_METADATA].tolist()
+
+ for timestamp, metadata in zip(timestamps, metadatas):
+ metadata_obj[timestamp] = json.loads(metadata)
+ metadata_obj[timestamp]['date'] = timestamp
+
+ return metadata_obj
+ # talp_outputs = df[TALP_DB_COLUMN_TALP_OUPUT].tolist()
+
+
+def pack_series_data(name, data):
+ return {
+ 'name': name,
+ 'type': 'line',
+ 'data': data}
+
+
+def extract_dataseries(df, metric):
+ timestamps = get_formatted_timestamps(df)
+ regions = extract_region_names_from_df(df)
+ talp_outputs = df[TALP_DB_COLUMN_TALP_OUPUT].tolist()
+ series = []
+
+ for region in regions:
+ data = []
+ for talp_output in talp_outputs:
+ raw_data = json.loads(talp_output)
+ for entry in raw_data['popMetrics']:
+ if entry['name'] == region:
+ try:
+ data.append(entry[metric])
+ except:
+ data.append(None)
+ print(
+ "WHOOPS not every timestamp has a data point, appending none")
+ if len(timestamps) != len(data):
+ print("WHOOPS not every timestamp has a data point")
+ series.append(pack_series_data(region, data))
+
+ print(series)
+ return series
+
+
+def main():
+ # Parse command-line arguments
+ parser = argparse.ArgumentParser(
+ description='Render HTML summary of the historic talp data')
+ parser.add_argument('input', help='Path to the TALP.db file')
+ parser.add_argument('-o', '--output', dest='output',
+ help='Name of the html file beeing generated. If not specified [input].html will be chosen', required=False)
+ args = parser.parse_args()
+
+ # Check if the JSON file exists
+ if not os.path.exists(args.input):
+ print(
+ f"Error: The specified SQLITE file '{args.input}' does not exist.")
+ return
+
+ # Set output
+ if args.output:
+ OUTPUT_FILE = args.output
+ if not args.output.endswith('.html'):
+ OUTPUT_FILE += ".html"
+ print(f"Info: Appending .html to '{args.output}'")
+ # Check if the HTML file exists
+ if os.path.exists(args.output):
+ print(f"Info: Overwriting '{args.output}'")
+ else:
+ OUTPUT_FILE = args.input.replace(".json", "")
+ OUTPUT_FILE += ".html"
+
+ # Read in the data
+ conn = sqlite3.connect(args.input)
+
+ df = pd.read_sql(f"SELECT * FROM {TALP_TABLE_NAME}", conn)
+
+ print(df)
+ region_names = extract_region_names_from_df(df)
+ metadata = extract_metadata_from_df(df)
+ pe_series = extract_dataseries(df, 'parallelEfficiency')
+ et_series = extract_dataseries(df, 'elapsedTime')
+ ipc_series = extract_dataseries(df, 'averageIPC')
+ timestamps = get_formatted_timestamps(df)
+
+ # Render the template with the data
+ rendered_html = render_template(TALP_TEMPLATE_PATH, 'talp_time_series.jinja',
+ timestamps=timestamps,
+ region_names=region_names,
+ metadata=metadata,
+ pe_series=pe_series,
+ et_series=et_series,
+ ipc_series=ipc_series,
+ default_region_name=TALP_DEFAULT_REGION_NAME)
+
+ # Save or display the rendered HTML as needed
+ with open(OUTPUT_FILE, 'w') as f:
+ f.write(rendered_html)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/talp_report/talp_variables.py b/talp_report/talp_variables.py
new file mode 100644
index 0000000000000000000000000000000000000000..628ae926dc7bf550d946dec062c4281ea716b1af
--- /dev/null
+++ b/talp_report/talp_variables.py
@@ -0,0 +1,14 @@
+"""
+File declaring some global scoped variables we rely to be there in our scrips
+"""
+import pathlib
+
+TALP_TABLE_NAME = "talp_data"
+TALP_DB_COLUMN_TALP_OUPUT = "talp_ouput"
+TALP_DB_COLUMN_TIMESTAMP = "timestamp"
+TALP_DB_COLUMN_METADATA = "metadata"
+TALP_DEFAULT_REGION_NAME = "MPI Execution"
+
+TALP_TEMPLATE_PATH = pathlib.Path(__file__).parent.resolve()
+TALP_TABLE_COLUMNS_WITH_DATATYPES = f"({TALP_DB_COLUMN_TIMESTAMP} TIMESTAMP, {TALP_DB_COLUMN_TALP_OUPUT} TEXT, {TALP_DB_COLUMN_METADATA} TEXT)"
+TALP_TABLE_COLUMNS = f"({TALP_DB_COLUMN_TIMESTAMP} ,{TALP_DB_COLUMN_TALP_OUPUT}, {TALP_DB_COLUMN_METADATA})"
+