cubism.js with graphite server

Recently i’d been looking for a better interface for displaying multiple server stats.
Low and behold i found one created by Square payments.
Here’s what this thing looks like

cubism_example

Their project and examples can be found here:
http://square.github.com/cubism/

I’m going to assume you have an apache server and graphite server running somewhere.

So here’s a small example of how to get this going and your metrics plotting.
It’s pretty much an html file with some javascript
Create an new html file in your apache docroot with the following information ( usually /var/www/ in most ubuntu systems )

1. Put in your page title and some css includes and js includes

Alot of this is going to pull the css and js source directly from Square’s page.
I would recommend you download these files and copy them locally for your own use and serving them from apache yourself.


<meta charset="utf-8" />
Cubism.js</pre>
<style>
@import url(//fonts.googleapis.com/css?family=Yanone+Kaffeesatz:400,700);
@import url(//square.github.com/cubism/style.css);
</style>
<div id="body">
<h2>Host01 Load Average</h2>
<div id="graphs"></div>
<script type="text/javascript" src="http://d3js.org/d3.v2.js"></script>
<script type="text/javascript" src="http://square.github.com/cubism/cubism.v1.js"></script>
<script type="text/javascript" src="http://square.github.com/cubism/highlight.min.js"></script>
<script type="text/javascript">
{font-family:arial,helvettica,sans-serif"}
</script>

2. Setup some cubism settings.

Set the time granularity with the “.step” setting
Set the number of metrics you want to display with the “.size” setting

<script type="text/javascript">
var context = cubism.context()
    .step( 1 * 60 * 1000 )   // 1 minute
    .size(960);  // 1 * 960 = 4 hours

Setup more cubism graphite settings

Set the address of your graphite webserver at the “context.graphite”
Set the height of each row of metric data at “.height”
Set the time shift of how many days you want to go back at “.shift”, like you want to see data from 7 days ago.

var graphite = context.graphite("http://graphite-server.foo-bar.net");
var horizon = context.horizon().metric(graphite.metric).height(100).shift( - 0 * 24 * 60 * 60 * 1000 );

Create a list of metrics you want to see in an array

var metrics = [
   'stats.host01.cpu.load.load',
   'stats.host02.cpu.load.load',
   'nonNegativeDerivative(stats.host02.network.eth0.interface_tx_bytes)'
]

Call d3 and apply a bunch of stuff to the div with the id=graphs

d3.select("#graphs").append("div")
    .attr("class", "axis")
    .call(context.axis().orient("top"));

d3.select("#graphs").append("div")
    .attr("class", "rule")
    .call(context.rule());

d3.select("#graphs").selectAll(".horizon")
    .data(metrics)
  .enter().append("div")
    .attr("class", "horizon")
    .call(horizon);
</script>

Put it all together

The fully constructed HTML file should look something like this


<meta charset="utf-8" />
Cubism.js</pre>
<style>
@import url(//fonts.googleapis.com/css?family=Yanone+Kaffeesatz:400,700);
@import url(//square.github.com/cubism/style.css);
</style>
<div id="body">
<h2>Host01 Load Average</h2>
<div id="graphs"></div>
<script type="text/javascript" src="http://d3js.org/d3.v2.js"></script>
<script type="text/javascript" src="http://square.github.com/cubism/cubism.v1.js"></script>
<script type="text/javascript" src="http://square.github.com/cubism/highlight.min.js"></script>
<script type="text/javascript">
{font-family:arial,helvettica,sans-serif"}
</script>

<script type="text/javascript">
var context = cubism.context()
    .step( 1 * 60 * 1000 )   // 1 minute
    .size(960);  // 1 * 960 = 4 hours

var graphite = context.graphite("http://graphite-server.foo-bar.net");
var horizon = context.horizon().metric(graphite.metric).height(100).shift( - 0 * 24 * 60 * 60 * 1000 );

var metrics = [
   'stats.host01.cpu.load.load',
   'stats.host02.cpu.load.load',
   'nonNegativeDerivative(stats.host02.network.eth0.interface_tx_bytes)'
]

d3.select("#graphs").append("div")
    .attr("class", "axis")
    .call(context.axis().orient("top"));

d3.select("#graphs").append("div")
    .attr("class", "rule")
    .call(context.rule());

d3.select("#graphs").selectAll(".horizon")
    .data(metrics)
  .enter().append("div")
    .attr("class", "horizon")
    .call(horizon);
</script>

A more complicated example using graphite.find function


<meta charset="utf-8" />
Cubism.js</pre>
<style>
@import url(//fonts.googleapis.com/css?family=Yanone+Kaffeesatz:400,700);
@import url(http://square.github.com/cubism/style.css);
</style>
<div id="body">
<h2>Host01 Load Average</h2>
<div id="graphs"></div>
<script type="text/javascript" src="http://d3js.org/d3.v2.js"></script>
<script type="text/javascript" src="http://square.github.com/cubism/cubism.v1.js"></script>
<script type="text/javascript" src="http://square.github.com/cubism/highlight.min.js"></script>
<script type="text/javascript">
{font-family:arial,helvettica,sans-serif"}
</script>

<script type="text/javascript">
var context = cubism.context()
    .step( 1 * 60 * 1000 )   // 1 minute
    .size(960);  // 1 * 960 = 4 hours

var graphite = context.graphite("http://graphite-server.foo-bar.net");
//////// Example: 'stats.host*.cpu.load.load'
graphFind = 'stats.host0*.network.eth*.interface_*_bytes'

// Set The Time Row on Top
d3.select("#graphs").append("div")
    .attr("class", "axis")
    .call(context.axis().orient("top"));

// Set the Vertical Line Bar
d3.select("#graphs").append("div")
    .attr("class", "rule")
    .call(context.rule());

graphite.find(graphFind, function(error, results) {

   // Map find results to array and set to graphite.metric object type
    var metrics = results.sort().map(function(i) {
      return graphite.metric(i);
      //// return it as a nonNegativeDerivative
      // return graphite.metric('nonNegativeDerivative('+i+')');
    });

   // loop through array and print stuff to "graphs" div and apply .height and .colors to object
   for (var i=0;i<metrics.length;i++){
    d3.select("#graphs").call(function(div) {
        div.append("div").selectAll(".horizon")
             .data([metrics[i]])
             .enter().append("div")
             .attr("class", "horizon")
            .call(context.horizon()
              .height(100)
              .colors(["#08519c","#3182bd","#6baed6","#bdd7e7","#bae4b3","#74c476","#31a354","#006d2c"])
            );
    });
   }
   // Set The Time Row on Bottom
   d3.select("#graphs").append("div")
       .attr("class", "axis")
       .call(context.axis().orient("bottom"));
});
</script>
Advertisement

Ruby – regex example

I thought i might throw out some simple examples of using regexes with ruby for when i forget

command = `mpstat -P ALL`
regex = /(?<NAME0>load)\s+average:\s+(?<NAME1>\S+),\s+(?<NAME2>\S+),\s+(?<NAME3>\S+)/x
result = command.match(regex)

# Print your regex
puts " #{result['NAME0']} #{result['NAME1']} #{result['NAME2']} #{result['NAME3']}"
#or
puts " #{result[1]} #{result[2]} #{result[3]} #{result[4]}"

annndd… something more complicated in context of something else


#!/usr/bin/env ruby
require "getopt/long"
require 'socket'

opt = Getopt::Long.getopts(
     ["--server", "-s", Getopt::REQUIRED],
     ["--port", "-p", Getopt::REQUIRED],
     ["--environment", "-e", Getopt::REQUIRED]
)

unless opt["s"] and opt["p"] and opt["e"]
  unless opt["p"] =~ /\d+/
    currentFile = File.basename(__FILE__)
    puts "usage: ./#{currentFile} -s graphiteServer -p graphitePort -e siteEnvironment"
    puts "usage: ./#{currentFile} -s someserver -p 2003 -e dev"
    exit 1
  end
end

statprefix = 'stats'
hostname = `hostname`.chomp
command = `mpstat -P ALL`
epoch = (Time.now.to_i).to_s
graphiteServer = opt["s"]
graphitePort = opt["p"]
siteEnv = opt["e"]

regexTitles = /(?<TITLEID>CPU\s.*)/x
partsTitle = command.match(regexTitles)
partsTitle = partsTitle['TITLEID'].split

regex = /(?<CPUID>all.*)/x
parts = command.match(regex)
parts = parts['CPUID'].split

hash = Hash[partsTitle.zip(parts)]
sock = TCPSocket.new(graphiteServer, graphitePort)
hash.each_pair do |title,value|
  title = title.sub(/^\%/,"")
  sock.puts "#{statprefix}.#{siteEnv}.#{hostname}.cpu.all.#{title} #{value} #{epoch}"
end
sock.close

Graphite – aggregating your second retention bucket

Make sure your storage-aggregator.py is running in order databases to compress down into 2nd and 3rd retention levels.

$ ls
league_number.wsp  user_number.wsp

Find out info on your database “league_number”
Seeing something like xFilesFactor .5 means that 50% of that time retention period has to show
if you have 10 second intervals but are only inputting once every 60 seconds then your xFilesFactor is only .2
Set the xFilesFactor = 0 if you don’t want to drop any of your data

$ /usr/local/bin/whisper-info.py league_number.wsp
maxRetention: 315360000
xFilesFactor: 0.5
aggregationMethod: average
fileSize: 63097960

Archive 0
retention: 21600
secondsPerPoint: 10
points: 2160
size: 25920
offset: 40

Archive 1
retention: 315360000
secondsPerPoint: 60
points: 5256000
size: 63072000
offset: 25960

Resize your whisper database if data is not being aggregated properly

$ /usr/local/bin/whisper-resize.py league_number.wsp
Usage: whisper-resize.py path timePerPoint:timeToStore [timePerPoint:timeToStore]*

timePerPoint and timeToStore specify lengths of time, for example:

60:1440      60 seconds per datapoint, 1440 datapoints = 1 day of retention
15m:8        15 minutes per datapoint, 8 datapoints = 2 hours of retention
1h:7d        1 hour per datapoint, 7 days of retention
12h:2y       12 hours per datapoint, 2 years of retention

If you have fucked up graphs that are truncating data on the second retention size
align them like so

$ /usr/local/bin/whisper-resize.py user_number.wsp --xFilesFactor=0 --aggregationMethod=average 10:6h 1min:10y

Make sure you set your storage-aggregation correctly and make sure it’s started

$ cp storage-aggregation.conf.example storage-aggregation.conf

$ /opt/graphite/bin/carbon-aggregator.py status
carbon-aggregator (instance a) is not running
$ /opt/graphite/bin/carbon-aggregator.py start
Starting carbon-aggregator (instance a)

Graphite Install

*this assumes debian or ubuntu

Graphite Cookbook Install

$ git clone https://github.com/cookingclouds/cookbooks/tree/master/cc_graphite

I Know there is a cookbook for graphite in the opscode repo but it has dependencies.
This cookbook shouldn’t have dependencies and is also a lot more messy
Run after installing cookbook :

$ /opt/graphite/bin/carbon-cache.py start
$ cd /opt/graphite/webapp/graphite && python manage.py createsuperuser

Manual Install

Grab the sourcecode


$ wget https://launchpad.net/graphite/0.9/0.9.10/+download/graphite-web-0.9.10.tar.gz
$ wget https://launchpad.net/graphite/0.9/0.9.10/+download/carbon-0.9.10.tar.gz
$ wget https://launchpad.net/graphite/0.9/0.9.10/+download/whisper-0.9.10.tar.gz
$ wget https://launchpad.net/graphite/0.9/0.9.10/+download/check-dependencies.py

Install Whisper Database

$ ls |grep tar |while read i ; do tar xzvf $i ; done
$ apt-get -y install apache2 python-django python-django-tagging python-ldap python-memcache python-cairo
$ cd ~/whisper-0.9.10/
$ python setup.py install

Install Carbon

$ apt-get -y install python-twisted python-simplejson
$ cd ~/carbon-0.9.10/
$ python setup.py install
$ cd /opt/graphite/conf/
$ cp carbon.conf.example carbon.conf
$ cp storage-schemas.conf.example storage-schemas.conf
$ cd /opt/graphite/
$ ./bin/carbon-cache.py start

Install Graphite-web

$ cd ~/graphite-web-0.9.10/
$ python check-dependencies.py
$ python setup.py install
$ cp examples/example-graphite-vhost.conf /etc/apache2/sites-available/default
$ cp /opt/graphite/conf/graphite.wsgi.example /opt/graphite/conf/graphite.wsgi
$ mkdir /etc/apache2/run
$ apt-get -y install libapache2-mod-wsgi
$ /etc/init.d/apache2 reload
$ cd /opt/graphite/webapp/graphite/
$ python manage.py syncdb
$ chown -R www-data:www-data /opt/graphite/storage
$ cd /opt/graphite/webapp/graphite
$ cp local_settings.py.example local_settings.py
$ /etc/init.d/apache2 restart

Examples on how to send data to graphite

hostname = * you guessed it
test_001 = service name
1001 = value
$NOW = epoch time to input in graph

$ NOW=`date +%s` ; echo "hostname.test_001 1001 $NOW" |nc localhost 2003

or

$ cd /opt/graphite/examples
$ ./example-client.py