Ruby – regex example

I thought i might throw out some simple examples of using regexes with ruby for when i forget

command = `mpstat -P ALL`
regex = /(?<NAME0>load)\s+average:\s+(?<NAME1>\S+),\s+(?<NAME2>\S+),\s+(?<NAME3>\S+)/x
result = command.match(regex)

# Print your regex
puts " #{result['NAME0']} #{result['NAME1']} #{result['NAME2']} #{result['NAME3']}"
#or
puts " #{result[1]} #{result[2]} #{result[3]} #{result[4]}"

annndd… something more complicated in context of something else


#!/usr/bin/env ruby
require "getopt/long"
require 'socket'

opt = Getopt::Long.getopts(
     ["--server", "-s", Getopt::REQUIRED],
     ["--port", "-p", Getopt::REQUIRED],
     ["--environment", "-e", Getopt::REQUIRED]
)

unless opt["s"] and opt["p"] and opt["e"]
  unless opt["p"] =~ /\d+/
    currentFile = File.basename(__FILE__)
    puts "usage: ./#{currentFile} -s graphiteServer -p graphitePort -e siteEnvironment"
    puts "usage: ./#{currentFile} -s someserver -p 2003 -e dev"
    exit 1
  end
end

statprefix = 'stats'
hostname = `hostname`.chomp
command = `mpstat -P ALL`
epoch = (Time.now.to_i).to_s
graphiteServer = opt["s"]
graphitePort = opt["p"]
siteEnv = opt["e"]

regexTitles = /(?<TITLEID>CPU\s.*)/x
partsTitle = command.match(regexTitles)
partsTitle = partsTitle['TITLEID'].split

regex = /(?<CPUID>all.*)/x
parts = command.match(regex)
parts = parts['CPUID'].split

hash = Hash[partsTitle.zip(parts)]
sock = TCPSocket.new(graphiteServer, graphitePort)
hash.each_pair do |title,value|
  title = title.sub(/^\%/,"")
  sock.puts "#{statprefix}.#{siteEnv}.#{hostname}.cpu.all.#{title} #{value} #{epoch}"
end
sock.close
Advertisements

Graphite – aggregating your second retention bucket

Make sure your storage-aggregator.py is running in order databases to compress down into 2nd and 3rd retention levels.

$ ls
league_number.wsp  user_number.wsp

Find out info on your database “league_number”
Seeing something like xFilesFactor .5 means that 50% of that time retention period has to show
if you have 10 second intervals but are only inputting once every 60 seconds then your xFilesFactor is only .2
Set the xFilesFactor = 0 if you don’t want to drop any of your data

$ /usr/local/bin/whisper-info.py league_number.wsp
maxRetention: 315360000
xFilesFactor: 0.5
aggregationMethod: average
fileSize: 63097960

Archive 0
retention: 21600
secondsPerPoint: 10
points: 2160
size: 25920
offset: 40

Archive 1
retention: 315360000
secondsPerPoint: 60
points: 5256000
size: 63072000
offset: 25960

Resize your whisper database if data is not being aggregated properly

$ /usr/local/bin/whisper-resize.py league_number.wsp
Usage: whisper-resize.py path timePerPoint:timeToStore [timePerPoint:timeToStore]*

timePerPoint and timeToStore specify lengths of time, for example:

60:1440      60 seconds per datapoint, 1440 datapoints = 1 day of retention
15m:8        15 minutes per datapoint, 8 datapoints = 2 hours of retention
1h:7d        1 hour per datapoint, 7 days of retention
12h:2y       12 hours per datapoint, 2 years of retention

If you have fucked up graphs that are truncating data on the second retention size
align them like so

$ /usr/local/bin/whisper-resize.py user_number.wsp --xFilesFactor=0 --aggregationMethod=average 10:6h 1min:10y

Make sure you set your storage-aggregation correctly and make sure it’s started

$ cp storage-aggregation.conf.example storage-aggregation.conf

$ /opt/graphite/bin/carbon-aggregator.py status
carbon-aggregator (instance a) is not running
$ /opt/graphite/bin/carbon-aggregator.py start
Starting carbon-aggregator (instance a)

Adding EBS Volumes with Opscode’s AWS cookbook

1. Download opscode’s aws cookbook and put it into your own cookbook repo

$ git clone https://github.com/opscode/cookbooks.git opscode-cookbooks
$ cp -r opscode-cookbooks/aws my-cookbooks/
$ cd my-cookbooks 

2. Create a new cookbook that will utilize the aws cookbook

$ knife cookbook create aws-tests

3. Set the cookbook to have the dependency of the opscode aws cookbook

$ vi my-cookbooks/aws-tests/metadata.rb
maintainer       "YOUR_COMPANY_NAME"
maintainer_email "YOUR_EMAIL"
license          "All rights reserved"
description      "Installs/Configures aws_tests"
long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
version          "0.0.1"

depends "aws"

4. Create your recipe to create and attach a new EBS volume to your ec2 instance

$ vi vi my-cookbooks/aws-tests/recipes/default.rb
# Create and attach your new EBS volume
aws_ebs_volume "new_ebs_volume" do
  aws_access_key "MYAPIKEY"
  aws_secret_access_key "MYAPIKEYSECRET"
  size 1
  device "/dev/xvdi"
  action [ :create, :attach ]
end

5. Create a filesystem and mount your new volume

# Create your partition and filesystem for ext4
bash "create_filesystem" do
  user "root"
  code <<-EOH
    parted /dev/xvdi mklabel gpt
    parted /dev/xvdi mkpart logical ext4 1 -1
    parted /dev/xvdi set 1 lvm on
    yes | parted /dev/xvdi mkpart logical ext4 1 -- "-1"
    mkfs.ext4 /dev/xvdi1
  EOH
  not_if "parted /dev/xvdi1 |grep ext4"
end


directory "/mnt/test" do
  owner "root"
  group "root"
  mode "0755"
  recursive true
end

mount "/mnt/test" do
  device "/dev/xvdi1"
  options "rw noatime"
  fstype "ext4"
  action [ :enable, :mount ]
  not_if "cat /proc/mounts |grep /mnt/test"
end

6. Add aws and aws_tests recipes to your node

$ knife node edit i-fff4f8c
{
  "chef_environment": "_default",
  "name": "i-fff4f8",
  "run_list": [
    "recipe[aws]",
    "recipe[aws_tests]"
  ],
  "normal": {
    "tags": [

    ],
  }
}

7. Run chef-client on your node

$ chef-client

Debugging knife configure -i

If you ever see something like this


root@chefserver01:/etc/chef# knife configure -i -V
Overwrite /root/.chef/knife.rb? (Y/N) y
Please enter the chef server URL: [http://chefserver01:4000]
Please enter a clientname for the new client: [root] jtran7
Please enter the existing admin clientname: [chef-webui]
Please enter the location of the existing admin client's private key: [/etc/chef/webui.pem]
Please enter the validation clientname: [chef-validator]
Please enter the location of the validation key: [/etc/chef/validation.pem]
Please enter the path to a chef repository (or leave blank):
Creating initial API user...
INFO: HTTP Request Returned 500 Internal Server Error: Connection reset by peer
ERROR: Server returned error for http://chefserver01:4000/clients, retrying 1/5 in 4s
INFO: HTTP Request Returned 409 Conflict: Client already exists
INFO: HTTP Request Returned 500 Internal Server Error: Connection reset by peer
ERROR: Server returned error for http://chefserver01:4000/clients/jtran7, retrying 1/5 in 4s
INFO: HTTP Request Returned 500 Internal Server Error: Connection reset by peer
ERROR: Server returned error for http://chefserver01:4000/clients/jtran7, retrying 2/5 in 5s

it’s likely your ampq password is incorrect in /etc/chef/server.rb

# amqp_pass sets the password for the AMQP virtual host in rabbitmq-server.
amqp_pass "testing"

If you had deleted your rabbitmq mnesia table you had to do something like this.
The word “testing” is your ampq password

sudo rabbitmqctl add_vhost /chef
sudo rabbitmqctl add_user chef testing
sudo rabbitmqctl set_permissions -p /chef chef ".*" ".*" ".*"

Clustered AMQP Rabbit-MQ

RabbitMQ already has excellent documentation at: http://www.rabbitmq.com/clustering.html

But this will be a bit more chef specific
server01 = existing chef-server
server02 = new rabbitmq server to be added to cluster

On Server01

On your existing chef-server
*I assume you’re using the latest rabbitmq and that server02 will also install matching version
Check what the cluster output looks like

root@server01:~# rabbitmqctl cluster_status
Cluster status of node rabbit@server01 ..
[{nodes,[{disc,[rabbit@server01]}]},{running_nodes,[rabbit@server01]}]
...done.

Copy your rabbitmq cookie to server02

root@server01:~# scp /var/lib/rabbitmq/.erlang.cookie root@server02:/var/lib/rabbitmq/
Are you sure you want to continue connecting (yes/no)? yes
root@server02's password: 
.erlang.cookie                                                                                        100%   20     0.0KB/s   00:00    

On Server02

Install rabbitmq

root@server02:~# echo "deb http://www.rabbitmq.com/debian/ testing main" |tee -a /etc/apt/sources.list
root@server02:~# wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc
root@server02:~# apt-key add rabbitmq-signing-key-public.asc
root@server02:~# apt-get update
root@server02:/var/chef/cache# apt-get -y install rabbitmq-server 

Delete existing mnesia database and start rabbitmq

root@server02:~# service rabbitmq-server stop
root@server02:~# rm -fr /var/lib/rabbitmq/mnesia 
root@server02:~# service rabbitmq-server start
 * Starting message broker rabbitmq-server
   ...done.

Join Server02 to Server01

root@server02:~# rabbitmqctl cluster_status
Cluster status of node rabbit@server02 ...
[{nodes,[{disc,[rabbit@server02]}]},{running_nodes,[rabbit@server02]}]
...done.
root@server02:~# rabbitmqctl stop_app
Stopping node rabbit@server02 ...
...done.
root@server02~# rabbitmqctl reset   
Resetting node rabbit@server02 ...
...done.
root@server02:~# rabbitmqctl cluster rabbit@server01 rabbit@server02
Clustering node rabbit@server02 with [rabbit@server01,rabbit@server02] ...
...done.
root@server02:~# rabbitmqctl start
root@server02:~# rabbitmqctl cluster_status
Cluster status of node rabbit@server01 ...
[{nodes,[{disc,[rabbit@server02,rabbit@server01]}]},
 {running_nodes,[rabbit@server02,rabbit@server01]}]
...done.

Verify on server01 that cluster shows up as expected

root@server01:~# rabbitmqctl cluster_status
Cluster status of node rabbit@server01 ...
[{nodes,[{disc,[rabbit@server02,rabbit@server01]}]},
 {running_nodes,[rabbit@server02,rabbit@server01]}]
...done.

Make sure chef still works

root@server01:~# knife node list

root@server01:~# knife client list
  chef-validator
  chef-webui
  root
  server01
root@server01:~# knife bootstrap -x root server02 --template-file ~/ubuntu12.04.rb 
Bootstrapping Chef on server02
Failed to authenticate root - trying password auth
Enter your password: 
server02 Updating installed gems

root@server01:~# knife node list
server02