05/11/2018 graph-stats.py 1 #!/usr/bin/env python

Size: px

Start display at page:

Download "05/11/2018 graph-stats.py 1 #!/usr/bin/env python"

Gerard Skinner
5 years ago
Views:

1 05/11/2018 graph-stats.py 1 #!/usr/bin/env python # 10/2018, C. Cervini, dbi-services; import sys import getopt from datetime import datetime import json import DctmAPI def Usage(): print(""" Usage: Connects as dmadmin/xxxx to a local repository and generates an HTML page containing the plots of several DQL queries result; Usage:./graph-stats.py -h --help -d --docbase <docbase>,<docbase [-o -- output_file <output_file>] <docbase> can be one repository or a comma-separated list of repositories; if <output_file> is omitted, the html page is output to./graphs/<docbase>-$(date +"%Y%m%d-%H%M%S").html; <output_file> can be "stdout", which is useful for CGI programming; Example:./graph-stats.py -d dmtest,mail_archive,doc_engineering will query the docbases dmtest, mail_archive and doc_engineering and output the graphs to the files <docbase>-$(date +"%Y%m%d-%H%M%S").html, one file per docbase;./graph-stats.py -d mail_archive,doc_engineering -o all_docbase_current_status will query the docbases mail_archive and doc_engineering and output all the graphs to the unique file all_docbase_current_status;./graph-stats.py -d mail-archive will query docbase mail-archive and output the graphs to the file mail-archive-$(date +"%Y%m%d-%H%M%S").html;./graph-stats.py --docbase dmtest --output dmtest.html will query docbase dmtest and output the graphs to the file dmtest.html;./graph-stats.py -d dmtest --output stdout will query docbase dmtest and output the graphs to stdout; """) def Plot2HTML(div, graph_title, data, data_labels, blinegraph = False, mode = None): global html_output if None!= html_output: sys.stdout = open(html_output, "a") # start the html page; if "b" == mode: global server, page_title <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2//EN"> <html> <head> <title>''' + page_title + '''</title> <meta http-equiv="content-type" content="text/html; charset=utf-8"> <script src=" + server + '''/plotly-latest.min.js"></script> </head> <body> <center><h3>''' + page_title + '''</h3></center> # append to the body of the html page; if len(data_labels) == 4: stack_labels = for point in data:

2 05/11/2018 graph-stats.py 2 stack_labels[point[data_labels[0]]] = 0 <div style="width:1500px;height:600px;"> <div id="''' + div + '-' + data_labels[2] + '''" style="width:50%; float:left;"></div> <div id="''' + div + '-' + data_labels[3] + '''" style="width:50%; float:left;"></div> </div> <script> var ctx = document.getelementbyid("''' + div + '-' + data_labels[2] + '''"); variables = "" for stack in stack_labels: x = [point[data_labels[1]] for i, point in enumerate(data) if point[data_labels[0]] == stack] y1 = [point[data_labels[2]] for i, point in enumerate(data) if point[data_labels[0]] == stack] variables += ("" if not variables else ", ") + "data_" + stack if blinegraph: vars()['data_' + stack] = 'name': stack, 'type': "scatter", 'mode': "lines", 'x': x, 'y': y1, ; else: vars()['data_' + stack] = 'name': stack, 'type': "bar", 'x': x, 'y': y1, 'width': 0.25, 'marker': #'color': '#009933', 'line': 'width': 1.0 ; print('data_' + stack + ' = ' + json.dumps(vars()['data_' + stack]) + ';') layout = 'title': '<b>' + graph_title + '<br>' + data_labels[2] + '</b>', 'legend': 'x': -.1, 'y': 1.2, 'font': 'size': 8, 'font': 'size': 12, 'width': 750, 'height': 600, 'xaxis': 'title': '<b>' + data_labels[1] + '</b>', 'titlefont': 'size': 10, 'tickangle': -45, 'tickfont': 'size': 8, 'showline': True, 'categoryorder': "category ascending", 'type': "category", 'yaxis': 'title': '<b>' + data_labels[2] + '</b>', 'titlefont': 'size': 10, 'showline': True,

3 05/11/2018 graph-stats.py 3 'tickfont': 'size': 8, 'showgrid': False if not blinegraph: layout.update('barmode': "stack", 'bargap': 0.15, 'bargroupgap': 0.5) interaction = 'responsive': True, 'scrollzoom': True, 'editable': True Plotly.newPlot(ctx, [''' + variables + '], ' + json.dumps(layout) + ',' + json.dumps(interaction) + ; for stack in stack_labels: x = [point[data_labels[1]] for i, point in enumerate(data) if point[data_labels[0]] == stack] y2 = [point[data_labels[3]] for i, point in enumerate(data) if point[data_labels[0]] == stack] if blinegraph: vars()['data_' + stack] = 'name': stack, 'type': "scatter", 'mode': "lines", 'x': x, 'y': y2 ; else: vars()['data_' + stack] = 'name': stack, 'type': "bar", 'x': x, 'y': y2, 'width': 0.25, 'marker': #'color': '#009933',*/ 'line': 'width': 1.0 ; print('data_' + stack + ' = ' + json.dumps(vars()['data_' + stack]) + ';') layout = 'title': '<b>' + graph_title + '<br>' + data_labels[3] + '</b>', 'legend': 'x': -.1, 'y': 1.2, 'font': 'size': 8, 'font': 'size': 12, 'width': 750, 'height': 600, 'xaxis': 'title': '<b>' + data_labels[1] + '</b>', 'titlefont': 'size': 10, 'tickangle': -45, 'tickfont': 'size': 8, 'showline': True, 'categoryorder': "category ascending", 'type': "category", 'yaxis': 'title': '<b>' + data_labels[3] + '</b>', 'titlefont': 'size': 10, 'showline': True,

4 05/11/2018 graph-stats.py 4 'tickfont': 'size': 8, 'showgrid': False if not blinegraph: layout.update('barmode': "stack", 'bargap': 0.15, 'bargroupgap': 0.5) interaction = 'responsive': True, 'scrollzoom': True, 'editable': True var ctx = document.getelementbyid("''' + div + '-' + data_labels[3] + '''"); Plotly.newPlot(ctx, [''' + variables + '],' + json.dumps(layout) + ''', ''' + json.dumps(interaction) + ; </script> elif len(data_labels) == 3: <div style="width:1200px;height:600px;"> <div id="''' + div + '''" style="width:75%; float:left;"></div> <div id="''' + div + '''-pie" style="width:25%; float:left;"></div> </div> <script> var ctx = document.getelementbyid("''' + div + '''"); traces = [] if not blinegraph: traces = [ 'name': data_labels[1], 'type': "bar", 'x': [point[data_labels[0]] for i, point in enumerate(data)], 'y': [point[data_labels[1]] for i, point in enumerate(data)], 'width': 0.25, 'marker': 'color': '#009933', 'line': 'width': 1.0,, # work around for bug "Grouped bar charts do not work with multiple Y axes"; # see # must be inserted here; # invisible second trace in the first group 'x': [point[data_labels[0]] for i, point in enumerate(data)], 'barmode': "overlay", 'y': [0], 'type': 'bar', 'hoverinfo': 'none', 'showlegend': False, # invisible first trace in the second group 'x': [point[data_labels[0]] for i, point in enumerate(data)], 'y': [0], 'type': 'bar', 'yaxis': 'y2', 'hoverinfo': 'none', 'showlegend': False, 'name': data_labels[2], 'type': "bar", 'x': [point[data_labels[0]] for i, point in enumerate(data)], 'y': [point[data_labels[2]] for i, point in enumerate(data)],

5 05/11/2018 graph-stats.py 5 ] else: traces = [ layout = ], 'width': 0.25, 'yaxis': "y2", 'marker': 'color': '#4e6bed', 'line': 'width': 1.0 'name': data_labels[1], 'type': "scatter", 'mode': "lines", 'x': [point[data_labels[0]] for i, point in enumerate(data)], 'y': [point[data_labels[1]] for i, point in enumerate(data)], 'width': 0.25, 'name': data_labels[2], 'type': "scatter", 'mode': "lines", 'x': [point[data_labels[0]] for i, point in enumerate(data)], 'y': [point[data_labels[2]] for i, point in enumerate(data)], 'width': 0.25, 'yaxis': "y2", 'title': '<b>' + graph_title + '</b>', 'legend': 'x': -.1, 'y': 1.2, 'font': 'size': 8, 'font': 'size': 12, 'width': 800, 'height': 600, 'xaxis': 'title': '<b>' + data_labels[0] + '</b>', 'titlefont': 'size': 10, 'tickangle': -45, 'tickfont': 'size': 8, 'showline': True, 'yaxis': 'title': '<b>' + data_labels[1] + '</b>', 'titlefont': 'size': 10, 'showline': True, 'tickfont': 'size': 8, 'showgrid': False,, 'yaxis2': 'title': '<b>' + data_labels[2] + '</b>', 'titlefont': 'size': 10, 'tickfont': 'size': 8, 'showline': True, 'overlaying': "y", 'side': "right", 'showgrid': False,

6 05/11/2018 graph-stats.py 6 if blinegraph: layout['yaxis'].update('type': "linear") layout['yaxis2'].update('type': "linear") pass else: layout.update('barmode': "group", 'bargap': 0.15, 'bargroupgap': 0.5) interaction = 'responsive': True, 'scrollzoom': True, 'editable': True Plotly.newPlot(ctx, ''' + json.dumps(traces) + ''', ''' + json.dumps(layout) + ''', ''' + json.dumps(interaction) + ; </script> if not blinegraph: traces = [ ] layout =,, 'name': data_labels[1], 'hole':.4, 'type': "pie", 'labels': [point[data_labels[0]] for i, point in enumerate(data)], 'values': [point[data_labels[1]] for i, point in enumerate(data)], 'domain': 'row': 0, 'column': 0, 'outsidetextfont': 'size': 8, 'insidetextfont': 'size': 8, 'legend': 'font': 'size': 8, 'textfont': 'size': 8, 'font': 'size': 8, 'hoverinfo': 'label+percent+name', 'hoverlabel': 'font': 'size': 8, 'textinfo': 'none' 'name': data_labels[2], 'hole':.4, 'type': "pie", 'labels': [point[data_labels[0]] for i, point in enumerate(data)], 'values': [point[data_labels[2]] for i, point in enumerate(data)], 'domain': 'row': 1, 'column': 0, 'outsidetextfont': 'size': 8, 'insidetextfont': 'size': 8, 'legend': 'font': 'size': 8, 'textfont': 'size': 8, 'font': 'size': 8, 'hoverinfo': 'label+percent+name', 'hoverlabel': 'font': 'size': 8, 'textinfo': 'none'

7 05/11/2018 graph-stats.py 7 'title': '<b>' + graph_title + '</b>', 'annotations': [ 'font': 'size': 8, 'showarrow': False, 'text': '<b>' + data_labels[1] + '</b>', 'x': 0.5, 'y': 0.8, 'font': 'size': 8, 'showarrow': False, 'text': '<b>' + data_labels[2] + '</b>', 'x': 0.5, 'y': 0.25 ], 'height': 600, 'width': 600, 'grid': 'rows': 2, 'columns': 1, 'legend': 'font': 'size': 10 <script> var ctx = document.getelementbyid("''' + div + '''-pie"); Plotly.newPlot(ctx, ''' + json.dumps(traces) + ''', ''' + json.dumps(layout) + ''' ); </script> elif len(data_labels) == 2: trace = [ 'name': data_labels[1], 'type': "bar", 'x': [point[data_labels[0]] for i, point in enumerate(data)], 'y': [point[data_labels[1]] for i, point in enumerate(data)], 'width': 0.25, 'marker': 'color': '#009933', 'line': 'width': 1.0 ] layout = 'title': '<b>' + graph_title + '</b>', 'legend': 'x': -.1, 'y': 1.2, 'font': 'size': 8, 'font': 'size': 12, 'height': 600, 'width': 800, 'xaxis': 'title': '<b>' + data_labels[0] + '</b>', 'titlefont': 'size': 10, 'tickangle': -45,

8 05/11/2018 graph-stats.py 8 'tickfont': 'size': 8,, 'yaxis': 'title': '<b>' + data_labels[1] + '</b>', 'titlefont': 'size': 10, 'showline': True, 'tickfont': 'size': 8, 'showgrid': False, 'barmode': "group", 'bargap': 0.15, 'bargroupgap': 0.5 interaction = 'responsive': True, 'scrollzoom': True, 'editable': True; <div style="width:1200px;height:600px;"> <div id="''' + div + '''" style="width:75%; float:left;"></div> <div id="''' + div + '''-pie" style="width:25%; float:left;"></div> </div> <script> var ctx = document.getelementbyid("''' + div + '''"); Plotly.newPlot(ctx, ''' + json.dumps(trace) + ''', ''' + json.dumps(layout) + ''', ''' + json.dumps(interaction) + ; </script> trace = [ ] layout =, 'name': data_labels[1], 'hole':.4, 'type': "pie", 'labels': [point[data_labels[0]] for i, point in enumerate(data)], 'values': [point[data_labels[1]] for i, point in enumerate(data)], 'domain': 'row': 0, 'column': 0, 'outsidetextfont': 'size': 8, 'insidetextfont': 'size': 8, 'legend': 'font': 'size': 8, 'textfont': 'size': 8, 'font': 'size': 8, 'hoverinfo': 'label+percent+name', 'hoverlabel': 'font': 'size': 8, 'textinfo': 'none' 'title': '<b>' + graph_title + '</b>', 'annotations': [ 'font': 'size': 8, 'showarrow': False, 'text': '<b>' + data_labels[1] + '</b>', 'x': 0.5,

9 05/11/2018 graph-stats.py 9, 'y': 0.5 ], 'height': 600, 'width': 600, 'grid': 'rows': 1, 'columns': 1, 'legend': 'font': 'size': 10 <script> var ctx = document.getelementbyid("''' + div + '''-pie"); Plotly.newPlot(ctx, ''' + json.dumps(trace) + ''', ''' + json.dumps(layout) + ; </script> else: print("illegal data_label value: " + repr(data_label)) # closes the html page; if "e" == mode: </body> </html> # restores default output stream; if None!= html_output: sys.stdout = sys. stdout def cumulandextend2(): """ for 2 variables in resultset; """ global rawdata sum_count = 0 sum_size = 0 new_month = str(datetime.now().year) + "/" + str(datetime.now().month) for ind, point in enumerate(rawdata): sum_count += int(point['count(*)']) point['count(*)'] = sum_count sum_size += int(point['tot_size']) point['tot_size'] = sum_size if rawdata[ind]['month'] < new_month: new_point = dict(rawdata[ind]) new_point['month'] = new_month rawdata.append(new_point) def cumulandextend3(key): """ for 3 variables in resultset; """ global rawdata prec_doc = "" sum_count = 0 sum_size = 0 new_month = str(datetime.now().year) + "/" + str(datetime.now().month) for ind, point in enumerate(rawdata): if "" == prec_doc: prec_doc = point[key] if point[key]!= prec_doc:

10 05/11/2018 graph-stats.py 10 # duplicate the last point so the line graphe shows a flat line and not a simple dot when there is a unique point for the document type; if rawdata[ind - 1]['month'] < new_month: new_point = dict(rawdata[ind - 1]) new_point['month'] = new_month rawdata.insert(ind, new_point) prec_doc = point[key] sum_count = 0 sum_size = 0 else: sum_count += int(point['count(*)']) point['count(*)'] = sum_count sum_size += int(point['tot_size']) point['tot_size'] = sum_size if rawdata[ind]['month'] < new_month: new_point = dict(rawdata[ind]) new_point['month'] = new_month rawdata.append(new_point) # # main; if name == " main ": DctmAPI.logLevel = 0 # parse the command-line parameters; # old-style for I don't need more flexibility here; repository = None output_file = None try: (opts, args) = getopt.getopt(sys.argv[1:], "hd:o:", ["help", "docbase=", "output="]) except getopt.getopterror: print("illegal option") print("./graph-stats.py -h --help [-d --docbase docbase] [-o --output_file output_file]") sys. for opt, arg in opts: if opt in ("-h", "--help"): Usage() sys.exit() elif opt in ("-d", "--docbase"): repository = arg elif opt in ("-o", "--output"): output_file = arg if None == repository: print("at least one repository must be specified") Usage() sys.exit() DctmAPI.show("Will connect to docbase(s): " + repository + " and output to " + ("stdout" if "stdout" == output_file else "one single file " + output_file if output_file is not None else "one file per docbase")) # needed to locally import the js library Plotly; server = " :8000" docbase_done = set() status = DctmAPI.dmInit() for pointer, docbase in enumerate(repository.split(",")): # graphe_1; if docbase in docbase_done: print("warning: docbase :s was already processed and won't be again,

11 05/11/2018 graph-stats.py 11 print("warning: docbase :s was already processed and won't be again, skipping...".format(docbase)) continue docbase_done.add(docbase) session = DctmAPI.connect(docbase = docbase, user_name = "dmadmin", password = "dmadmin") if session is None: print("no session opened, exiting...") page_title = "Graphical Overview of Repository " + docbase if None == output_file: html_output = "./graphs/" + docbase + "-" + datetime.today().strftime("%y%m %d-%h%m%s") + ".html" elif "stdout" == output_file: html_output = None else: html_output = output_file graph_name = "graphe_1" stmt = """select r_object_type, count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) group by r_object_type order by 1""" graph_title = "Q1. Present Count & Size Per Document Type", data_labels = attr_name, mode = ("b" if (("stdout" == output_file or output_file is not None) and 0 == pointer) or output_file is None else None)) # graphe_2; graph_name = "graphe_2" stmt = """select r_object_type, datetostring(r_creation_date, 'yyyy/mm') as "month", count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) group by r_object_type, datetostring(r_creation_date, 'yyyy/mm') order by 1, 2""" graph_title = "Q2. Monthly New Documents per Type", cumulandextend3('r_object_type') Plot2HTML(div = graph_name + "l", graph_title = "Q2l. Cumulated Monthly Documents per Type", data_labels = attr_name, blinegraph = True)

12 05/11/2018 graph-stats.py 12 # graphe_3; graph_name = "graphe_3" stmt = """select r_object_type, datetostring(r_modify_date, 'yyyy/mm') as "month", count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) where r_creation_date < r_modify_date group by r_object_type, datetostring(r_modify_date, 'yyyy/mm') order by 1, 2""" graph_title = "Q3. Monthly Modified Documents per Type", cumulandextend3('r_object_type') Plot2HTML(div = graph_name + "l", graph_title = "Q3l. Cumulated Monthly Modified Documents per Type", data_labels = attr_name, blinegraph = True) # graphe_4; graph_name = "graphe_4" stmt = """select datetostring(r_creation_date, 'yyyy/mm') as "month", count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) group by datetostring(r_creation_date, 'yyyy/mm') order by 1""" graph_title = "Q4. Monthly New Documents", data_labels = attr_name ) cumulandextend2() Plot2HTML(div = graph_name + "l", graph_title = "Q4l. Cumulated Monthly Documents", data_labels = attr_name, blinegraph = True) # graphe_5; graph_name = "graphe_5" stmt = """select datetostring(r_modify_date, 'yyyy/mm') as "month", count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) where r_creation_date!= r_modify_date group by datetostring(r_modify_date, 'yyyy/ mm') order by 1; """

13 05/11/2018 graph-stats.py 13 graph_title = "Q5. Monthly Modified Documents", cumulandextend2() Plot2HTML(div = graph_name + "l", graph_title = "Q5l. Cumulated Monthly Modified Documents", data_labels = attr_name, blinegraph = True, mode = 'e') # graphe_6; graph_name = "graphe_6" stmt = """select a_content_type, count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) group by a_content_type order by 1""" graph_title = "Q6. Count & Size Per Content Format", # graphe_7; graph_name = "graphe_7" stmt = """select a_content_type, datetostring(r_creation_date, 'yyyy-mm') as "month", count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) group by a_content_type, datetostring(r_creation_date, 'yyyy-mm') having count(*) > 10 order by 1, 2""" graph_title = "Q7. Monthly Created Documents Per Content Format with count(*) > 10", cumulandextend3('a_content_type') Plot2HTML(div = graph_name + "l", graph_title = "Q7l. Cumulated Monthly Created Documents Per Content Format with count(*) > 10", data_labels = attr_name, blinegraph = True) # graphe_8; graph_name = "graphe_8"

14 05/11/2018 graph-stats.py 14 stmt = """select a_storage_type, count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) group by a_storage_type order by a_storage_type""" graph_title = "Q8. Count & Size Per Filestore", # graphe_9; graph_name = "graphe_9" stmt = """select r_object_type, a_storage_type, count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) group by r_object_type, a_storage_type order by r_object_type, a_storage_type""" graph_title = "Q9. Count & Size Per Document Type & Filestore", # graphe_10; graph_name = "graphe_10" stmt = """select a_content_type, a_storage_type, count(*), sum(r_full_content_size) as "tot_size" from dm_document(all) group by a_content_type, a_storage_type having count(*) > 10 order by a_content_type, a_storage_type""" graph_title = "Q10. Count & Size Per Content Format & Filestore with count(*) > 10", # graphe_11; graph_name = "graphe_11" stmt = """select owner_name, count(*) from dm_acl group by owner_name order by 2 desc"""

15 05/11/2018 graph-stats.py 15 graph_title = "Q11. ACLs per owner", # graphe_12; graph_name = "graphe_12" stmt = """select acl_name, count(*) from dm_document(all) where acl_name not like 'dm_%' group by acl_name having count(*) >= 10 order by 1""" graph_title = "Q12. External ACLs in Use by >= 10 documents", # graphe_13; graph_name = "graphe_13" stmt = """select name, count(*) from dmi_queue_item group by name order by 2 desc""" graph_title = "Q13. Queue Items by Name", data_labels = attr_name, mode = ('e' if (("stdout" == output_file or output_file is not None) and len(repository.split(",")) - 1 == pointer) or output_file is None else None)) status = DctmAPI.disconnect(session) print("error while disconnecting")

Week 8. Big Data Analytics Visualization with plotly for R

Week 8. Big Data Analytics Visualization with plotly for R Hyeonsu B. Kang hyk149@eng.ucsd.edu May 2016 1 Adding interactivity to graphs Plotly is a collaboration platform for modern data science. It lets