{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 1, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": false, "title": "Go to data", "tooltip": "Go to data", "type": "link", "url": "http://transformers-benchmarks.hf.co/d/fdz33iyzln9c0a/transformers-benchmarks?orgId=1&from=${StartTime}&to=${EndTime}" } ], "liveNow": true, "panels": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "left", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "gpu_name" }, "properties": [ { "id": "custom.width", "value": 202 } ] }, { "matcher": { "id": "byName", "options": "left" }, "properties": [ { "id": "custom.width", "value": 407 } ] }, { "matcher": { "id": "byName", "options": "commit_message" }, "properties": [ { "id": "custom.width", "value": 524 } ] }, { "matcher": { "id": "byName", "options": "commit_id" }, "properties": [ { "id": "custom.width", "value": 353 } ] }, { "matcher": { "id": "byName", "options": "model_id" }, "properties": [ { "id": "custom.width", "value": 216 } ] } ] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, "id": 5, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "11.2.2", "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT commit_id, commit_message, metadata->>'gpu_name' as gpu_name, metadata->>'model_id' as model_id, created_at AS date FROM benchmarks WHERE branch = '${branch}' AND metadata->>'gpu_name' = '${gpu_name}' ORDER BY benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [ { "name": "commit_id", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_name", "type": "functionParameter" } ], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50, "whereJsonTree": { "children1": [ { "id": "baaa8aaa-89ab-4cde-b012-31922f96de3f", "properties": { "field": "commit_id", "fieldSrc": "field", "operator": "equal", "value": [ "${commit}" ], "valueError": [ null ], "valueSrc": [ "value" ], "valueType": [ "text" ] }, "type": "rule" } ], "id": "bab88a98-0123-4456-b89a-b1922f7d4f11", "type": "group" }, "whereString": "commit_id = '${commit}'" }, "table": "benchmarks" } ], "transparent": true, "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 13, "panels": [], "title": "Eager Forward Pass", "type": "row" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 7 }, "id": 7, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "pluginVersion": "11.2.2", "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'first_eager_forward_pass_time_secs' AS double precision) AS first_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "First eager forward pass", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 7 }, "id": 9, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'second_eager_forward_pass_time_secs' AS double precision) AS second_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Second eager forward pass", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, "id": 16, "panels": [], "title": "Time to next token", "type": "row" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 19 }, "id": 17, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'time_to_first_token_secs' AS double precision) AS time_to_first_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Time to first token", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 19 }, "id": 18, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'time_to_second_token_secs' AS double precision) AS time_to_second_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Time to second token", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 30 }, "id": 19, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'time_to_third_token_secs' AS double precision) AS time_to_third_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Time to third token", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 30 }, "id": 20, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'time_to_next_token_mean_secs' AS double precision) AS time_to_next_token_mean_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Time to subsequent next tokens mean", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "id": 14, "panels": [], "title": "Compiled Generate", "type": "row" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 42 }, "id": 8, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "always", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'first_compile_generate_time_secs' AS double precision) AS first_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "First compile generate", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 42 }, "id": 10, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'second_compile_generate_time_secs' AS double precision) AS second_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Second compile generate", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 53 }, "id": 11, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'third_compile_generate_time_secs' AS double precision) AS third_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Third compile generate", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-YlBl" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 53 }, "id": 12, "options": { "barRadius": 0.05, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "auto", "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT CAST(m.measurements->'fourth_compile_generate_time_secs' AS double precision) AS fourth_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Fourth compile generate", "transformations": [ { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "time" } ] } } ], "transparent": true, "type": "barchart" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 64 }, "id": 15, "panels": [ { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": 60000, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 65 }, "id": 1, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT\n d.cpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}';", "refId": "A", "sql": { "columns": [ { "parameters": [ { "name": "cpu_util", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "mem_megabytes", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_util", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_mem_megabytes", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "\"time\"", "type": "functionParameter" } ], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50, "whereJsonTree": { "children1": [ { "id": "baa888b8-89ab-4cde-b012-31922f8671e9", "properties": { "field": "commit_id", "fieldSrc": "field", "operator": "equal", "value": [ "${commit}" ], "valueError": [ null ], "valueSrc": [ "value" ], "valueType": [ "text" ] }, "type": "rule" } ], "id": "bab88a98-0123-4456-b89a-b1922f7d4f11", "type": "group" }, "whereString": "commit_id = '${commit}'" }, "table": "measurements" } ], "title": "CPU Utilization", "transparent": true, "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": 60000, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 65 }, "id": 4, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT\n b.commit_id,\n d.gpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}';", "refId": "A", "sql": { "columns": [ { "parameters": [ { "name": "cpu_util", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "mem_megabytes", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_util", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_mem_megabytes", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "\"time\"", "type": "functionParameter" } ], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50, "whereJsonTree": { "children1": [ { "id": "baa888b8-89ab-4cde-b012-31922f8671e9", "properties": { "field": "commit_id", "fieldSrc": "field", "operator": "equal", "value": [ "${commit}" ], "valueError": [ null ], "valueSrc": [ "value" ], "valueType": [ "text" ] }, "type": "rule" } ], "id": "bab88a98-0123-4456-b89a-b1922f7d4f11", "type": "group" }, "whereString": "commit_id = '${commit}'" }, "table": "measurements" } ], "title": "GPU Utilization", "transparent": true, "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": 60000, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decmbytes" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 74 }, "id": 2, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT d.mem_megabytes, d.time FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}';", "refId": "A", "sql": { "columns": [ { "parameters": [ { "name": "cpu_util", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "mem_megabytes", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_util", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_mem_megabytes", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "\"time\"", "type": "functionParameter" } ], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50, "whereJsonTree": { "children1": [ { "id": "baa888b8-89ab-4cde-b012-31922f8671e9", "properties": { "field": "commit_id", "fieldSrc": "field", "operator": "equal", "value": [ "${commit}" ], "valueError": [ null ], "valueSrc": [ "value" ], "valueType": [ "text" ] }, "type": "rule" } ], "id": "bab88a98-0123-4456-b89a-b1922f7d4f11", "type": "group" }, "whereString": "commit_id = '${commit}'" }, "table": "measurements" } ], "title": "Memory usage", "transparent": true, "type": "timeseries" }, { "datasource": {}, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": 60000, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decmbytes" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 74 }, "id": 3, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT\n d.gpu_mem_megabytes,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}';", "refId": "A", "sql": { "columns": [ { "parameters": [ { "name": "cpu_util", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "mem_megabytes", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_util", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "gpu_mem_megabytes", "type": "functionParameter" } ], "type": "function" }, { "parameters": [ { "name": "\"time\"", "type": "functionParameter" } ], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50, "whereJsonTree": { "children1": [ { "id": "baa888b8-89ab-4cde-b012-31922f8671e9", "properties": { "field": "commit_id", "fieldSrc": "field", "operator": "equal", "value": [ "${commit}" ], "valueError": [ null ], "valueSrc": [ "value" ], "valueType": [ "text" ] }, "type": "rule" } ], "id": "bab88a98-0123-4456-b89a-b1922f7d4f11", "type": "group" }, "whereString": "commit_id = '${commit}'" }, "table": "measurements" } ], "title": "GPU memory usage", "transparent": true, "type": "timeseries" } ], "title": "Usage metrics", "type": "row" } ], "schemaVersion": 39, "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "main", "value": "main" }, "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "definition": "SELECT DISTINCT branch FROM benchmarks;", "description": "", "hide": 0, "includeAll": false, "label": "branch", "multi": false, "name": "branch", "options": [], "query": "SELECT DISTINCT branch FROM benchmarks;", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": false, "text": "1729701492845", "value": "1729701492845" }, "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "definition": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = '${branch}' ORDER BY benchmark_id ASC LIMIT 1;", "description": "", "hide": 2, "includeAll": false, "multi": false, "name": "StartTime", "options": [], "query": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = '${branch}' ORDER BY benchmark_id ASC LIMIT 1;", "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": false, "text": "1730393397577", "value": "1730393397577" }, "datasource": { "default": true, "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "definition": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}' ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;", "description": "", "hide": 2, "includeAll": false, "multi": false, "name": "EndTime", "options": [], "query": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}' ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": false, "text": "NVIDIA A10G", "value": "NVIDIA A10G" }, "datasource": { "type": "grafana-postgresql-datasource", "uid": "be28nkzirtb0gd" }, "definition": "SELECT DISTINCT metadata->>'gpu_name' FROM benchmarks;", "description": "", "hide": 0, "includeAll": false, "label": "GPU", "multi": false, "name": "gpu_name", "options": [], "query": "SELECT DISTINCT metadata->>'gpu_name' FROM benchmarks;", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": true, "text": "10", "value": "10" }, "description": "The number of commits to display, going from most recent to the nth commit.", "hide": 0, "label": "Last # of commits", "name": "last_n_commits", "options": [ { "selected": true, "text": "10", "value": "10" } ], "query": "10", "skipUrlSync": false, "type": "textbox" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "hidden": false }, "timezone": "browser", "title": "Transformers benchmarks", "uid": "fdz33iyzln9c0a", "version": 10, "weekStart": "" }