Posts

Showing posts from January, 2012

[Level 1] Install maven in Ubuntu

In Ubuntu, you could use apt-get to install maven. # apt-get -y install maven2 Wish this helps. regards, Stanley Huang

[Level 1] Install git in Ubuntu

If you want to install git in Ubuntu, you could use apt-get command: # apt-get -y install git-core git-doc git-gui gitk Wish this helps. regards, Stanley Huang

[Level 3] Cassandra stress tool from git in Ubuntu.

If you want to do the stress test for Cassandra. You could try an open source project call cassandra-stress of zznate. First you must to have git client and maven. Please reference here: git installation , maven installation Clone the src from git. # git clone https://github.com/zznate/cassandra-stress.git Then you would find a folder named "cassandra-stress", cd it and build the binary. # cd ./cassandra-stress # mvn install and mvn would create a sub folder "target", cd it and run the stress script. # cd ./target/appassembler/ # sh ./bin/stress -o insert -b 1000 -n 2000000 localhost:9160 Wish this helps. regards, Stanley Huang

[Level 3] Run python in shell script.

If you want to wrap python script into shell script. You could use the following sample code. #!/bin/sh # -*- mode: Python -*- """:" # bash code here; finds a suitable python interpreter and execs this file. # prefer unqualified "python" if suitable: python -c 'import sys; sys.exit(sys.hexversion < 0x020500b0)' 2>/dev/null \ && exec python "$0" "$@" for pyver in 2.6 2.7 2.5; do which python$pyver > /dev/null 2>&1 && exec python$pyver "$0" "$@" done echo "No appropriate python interpreter found." >&2 exit 1 ":""" import os, sys print os.path.dirname('/Hello/World/c.txt') $ ./test.sh /Hello/World Wish this helps. regards, Stanley Huang

[Level 2] Cassandra token generation tool by Python.

#!/bin/env python import sys if (len(sys.argv) > 1): num=int(sys.argv[1]) else: num=int(raw_input("How many nodes are in your cluster? ")) for i in range(0, num): print 'node %d: %d' % (i, (i*(2**127)/num)) $ ./cassandraTokenGenTools.py How many nodes are in your cluster? 5 node 0: 0 node 1: 34028236692093846346337460743176821145 node 2: 68056473384187692692674921486353642291 node 3: 102084710076281539039012382229530463436 node 4: 136112946768375385385349842972707284582 Wish this helps. regards, Stanley Huang

[Level 2] JSON module in Python.

#!/bin/env python import json import random data = {} for i in range(0,2): key = 'i%s' % i data[key] = {} for j in range(0,3): subkey = 'j%s' % j data[key][subkey] = {} for k in range(0,4): item = 'k%s' % k value = int(random.random() * 100) data[key][subkey][item] = value print json.dumps(data) $ ./testPython.py {"i1": {"j0": {"k3": 93, "k2": 98, "k1": 0, "k0": 3}, "j1": {"k3": 53, "k2": 28, "k1": 99, "k0": 69}, "j2": {"k3": 3, "k2": 93, "k1": 74, "k0": 28}}, "i0": {"j0": {"k3": 67, "k2": 98, "k1": 15, "k0": 92}, "j1": {"k3": 70, "k2": 22, "k1": 63, "k0": 3}, "j2": {"k3": 88, "k2": 28, "

[Level 3] Add row count in select of MySQL.

mysql> set @row_count; mysql> select *, @row_count := @row_count + 1 as row_count from test; +------+---------+-----------+ | id | product | row_count | +------+---------+-----------+ | 1 | cloth | 1 | | 2 | shoes | 2 | | 3 | paints | 3 | +------+---------+-----------+ 10 rows in set (0.00 sec) Wish this helps. regards, Stanley Huang

[Level 3] rw_ether_atob, rw_ether_btoa stored function in MySQL.

delimiter // drop function if exists rw_ether_atob// create function rw_ether_atob(sAscii char(17)) returns bit(48) deterministic begin declare bReturn bit(48); ##set bReturn=conv(replace(sAscii,':',''),16,2); ## string output not bit ##set bReturn=conv(replace(sAscii,':',''),16,10); ## not work ##set bReturn=cast(replace(sAscii,':','') as bit(48)); ## syntax error ##set bReturn=bin(replace(sAscii,':','')); ## syntax error set bReturn=unhex(replace(sAscii,':','')); return bReturn; end// drop function if exists rw_ether_btoa// create function rw_ether_btoa(sBit bit(48)) returns char(17) deterministic begin declare sReturn char(17); set sReturn=lpad(hex(sBit),12,'0'); set sReturn=concat_ws(':', substr(sReturn,1,2), substr(sReturn,3,2), substr(sReturn,5,2), substr(sReturn,7,2), substr(sReturn,9,2), substr(sReturn,11,2)); return sReturn; end// delimiter ; /* mysql> c

[Level 1] Notes for MySQL. (config)

## take care with signature '???' [mysql] user=root password database=mysql #!include /etc/my.inc #prompt=\R:\m \d> #prompt=\u:\h \d> [mysqldump] [client] ## new cert #ssl-key=/usr/local/mysql/newcerts/client-key.pem #ssl-cert=/usr/local/mysql/newcerts/client-cert.pem #ssl-ca=/usr/local/mysql/newcerts/ca-cert.pem user=root host=localhost protocol=tcp compress password ######################################################[mysqld_multi] [mysqld_multi] server-id = 0 mysqld = /usr/local/mysql/bin/mysqld_safe mysqladmin = /usr/local/mysql/bin/mysqladmin user = multi_admin password = my_password [mysqld2] socket = /tmp/mysql.sock2 port = 3307 pid-file = /usr/local/mysql/data2/hostname.pid2 datadir = /usr/local/mysql/data2 language = /usr/local/mysql/share/mysql/english user = unix_user1 [mysqld3] mysqld = /path/to/safe_mysqld/safe_mysqld ledir = /path/to/mysqld-binary/ mysqladmin = /path/to/mysqladmin/mysqladmin socket = /

[Level 1] Testing notes for MySQL. (mysql commands)

#!/usr/bin/bash #ln -s /dev/null .mysql_history ## in my.cnf MYSQL_HISTFILE=/dev/null #export MYSQL_PS1="(\u@\h) [\d]> " ## mysql --prompt="(\u@\h) [\d]> " #mysql -u mysql mysql < ./test_mysql.sql #mysql -u root mysql < ./test_mysql.sql #mysql -u root --defaults-file=/etc/my.ini ## mysqlslap, benchmark test # mysqlslap -auto-generate-sql # use default script mysqlslap -a -c100 i10 -u root -p --engine=InnoDB,MyISAM,MEMORY,ARCHIVE ## mysqlcheck, for MyISAM, InnoDB, CSV and Archive. #mysqlcheck world; #mysqlcheck world City Country; #mysqlcheck --databases world test; #mysqlcheck --all-databases; ## bin-log #mysqlbinlog bin.000001 bin.000002 | more #mysqlbinlog bin.000001 bin.000002 | mysql #mysqlbinlog --start-datetime="2009-01-01 00:00:00" --stop-datetime="2009-01-01 23:59:59" bin.000001 bin.000002 | mysql #mysqlbinlog --start-position=1001 --stop-position=1999 bin.000001 bin.000002 | mysql ## purge binary logs #mysql> purge bin

[Level 1] Testing notes for MySQL. (sql commands)

The following commands are my testing notes for MySQL. set sql_mode:=''; -- set sql_mode='only_full_group_by'; -- with group by syntax, select columns must be included in group by list. set sql_mode='traditional'; -- a set of modes, fail sql transaction while datatype error or out of range. -- set sql_mode='pipes_as_concat'; -- set || from logic 'or' to string concatenation (standard sql). -- set sql_mode='ansi_quote'; -- ' for string, " for column name select @@sql_mode; -- set sql_mode='no_auto_create_user'; -- for not auto create user when grant, except grant with identified by. ######################################### notes ## select @@query_cache_type ## @@query_cache_size ## show variable like 'query%'; ## show status like 'qcache%'; ## select sql_no_cache ... /* mysql client */ /* \c to cancle mysql statement. */ /* command */ /* show processlist; show full processlist; show

[Level 3] Benchmark in MySQL

#!/usr/bin/bash #mysqlslap -a -c100 i10 -u root -p --engine=InnoDB,MyISAM,MEMORY,ARCHIVE mysqlslap -a --only-print ## ref.@@pseudo_thread_id to get different values of thread. # mysqlslap --create="CREATE TABLE A (a int);INSERT INTO A values (@@pseudo_thread_id)" # --query="SELECT * FROM A" --concurrency=50 --iterations=200 Wish this helps. regards, Stanley Huang

[Level 3] Transaction in MySQL.

#!/usr/bin/bash -vx MYSQL_HOME=/opt/mysql/mysql mysql -uroot -proot_pwd <<EOF create database if not exists test; use test; create table if not exists test (id int) engine=InnoDB; alter table test engine=Innodb; truncate table test; insert into test values (1),(2),(3); system cp $MYSQL_HOME/data/test/test.ibd $MYSQL_HOME/data/test/test.ibd.bak start transaction; insert into test values (4); system cp $MYSQL_HOME/data/test/test.ibd $MYSQL_HOME/data/test/test.ibd.tran1; rollback; system cp $MYSQL_HOME/data/test/test.ibd $MYSQL_HOME/data/test/test.ibd.rollback; system diff $MYSQL_HOME/data/test/test.ibd.tran1 $MYSQL_HOME/data/test/test.ibd.rollback > ./diff_tran1_rollback; start transaction; insert into test values (4); system cp $MYSQL_HOME/data/test/test.ibd $MYSQL_HOME/data/test/test.ibd.tran2 commit; system cp $MYSQL_HOME/data/test/test.ibd $MYSQL_HOME/data/test/test.ibd.commit system diff $MYSQL_HOME/data/test/test.ibd.tran2 $MYSQL_HOME/data/test/test.ibd.commit > ./di

[Level 3] XML in MySQL.

#!/usr/bin/bash createTable() { mysql -ujoseph <<EOF CREATE TABLE persons ( id int auto_increment primary key, data text ); EOF } #createTable createUsers(){ mysql -ujoseph <<EOF INSERT INTO persons(data) values (' <person> <name>stanley</name> <sex>m</sex> <addr>taipei</addr> <tels> <tel>12340001</tel> <tel>12340002</tel> <tel>12340003</tel> </tels> </person> '), (' <person> <name>joseph</name> <sex>m</sex> <addr>taipei</addr> <tels> <tel>12350001</tel> <tel>12350002</tel> <tel>12350003</tel> </tels> </person> ') ; EOF } #createUsers queryUserTels() { mysql -ujoseph <<EOF select ExtractValue(data,'//person/tels/tel[1]'), ExtractValue(data,'/person/tels/tel[2]'), ExtractValue(data,'person/tel

[Level 2] How to delete duplicate data in MySQL.

drop table if exists abc; create table abc (i int DEFAULT NULL); insert into abc values (1),(2),(2),(3),(3),(3),(NULL),(NULL),(NULL),(NULL); -- insert into abc values (),(),(),(); -- work -- insert into abc values (1),(),(),(),(); -- not work -- ERROR 1136 (21S01): Column count doesn't match value count at row 2 -- case 1: delete abc from abc, (select * from abc group by i having count(*)>1) as xyz where abc.i=xyz.i or (abc.i is NULL and xyz.i is NULL) limit 1; -- case 2: delete abc from abc, (select * from abc group by i having count(*)>1) as xyz where abc.i=xyz.i or (abc.i is NULL and xyz.i is NULL); /* result: mysql> -- case 1: mysql> delete abc from abc, (select * from abc group by i having count(*)>1) as xyz where abc.i=xyz.i or (abc.i is NULL and xyz.i is NULL) limit 1; ERROR 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'limit 1' at line 1 mysql>

[Level 2] mysqlshow notes for MySQL.

#!/usr/bin/bash -vx read -p "Enter Password: " p c="mysqlshow -uroot -p$p" case1() { c1="$c" c2="$c1 mysql" c3="$c2 user" c4="$c3 host" echo $c1 && $c1 echo $c2 && $c2 echo $c3 && $c3 echo $c4 && $c4 } case2() { c1="$c" c2="$c1 m%" c3="$c1 mysql t%" c4="$c1 mysql time_zone_transition Tr%" echo $c1 && $c1 echo $c2 && $c2 echo $c3 && $c3 echo $c4 && $c4 } case2 Wish this helps. regards, Stanley Huang

[Level 3] mysqldump/mysqlimport notes for MySQL.

#!/usr/bin/bash #dData=`dirname $0`/data dData=/tmp/data mkdir -p $dData && cd $dData mysqldump -uroot test > $dData/test.sql mysqldump -uroot test t > $dData/test.t.sql #mysql -uroot < $dData/test.sql #mysql -uroot < $dData/test.t.sql exit exit exit --routines --triggers #!/usr/bin/bash #dData=`dirname $0`/data dData=/tmp/data mkdir -p $dData cd $dData mysql -uroot <<EOF use test; truncate table t; select * from t; EOF mysqlimport --fields-terminated-by=, --fields-enclosed-by='"' \ --lines-terminated-by="\n", --ignore \ test $dData/t.txt ## '--ignore/--replace' contain unique key values already in the table, 'test' is database name, 't' is the tablename. Wish this helps. regards, Stanley Huang

[Level 3] Notes for MySQL replication

#################################################### master server set server_id=0; ## master server_id must be 0 grant replication slave on *.* to 'slave'@'%' identified by 'slave'; show master status; # +-----------------------+----------+--------------+------------------+ # | File | Position | Binlog_Do_DB | Binlog_Ignore_DB | # +-----------------------+----------+--------------+------------------+ # | Stanley-NB-bin.000103 | 377 | | | # +-----------------------+----------+--------------+------------------+ ## in my.cnf [mysqld] server-id=0 #################################################### slaver server set server_id=1; # shell> mysqldump -uroot -p test t1 > /tmp/t.sql # mysql> create database if not exist test; # mysql> use test; # mysql> source /tmp/t.sql change master to master_host='192.168.1.100', master_user='slave', master_password='slave', master_

[Level 2] CSV storage engine testing for MySQL.

For testing CSV storage engine for MySQL, I wrote a testing script as the following. #!/usr/bin/bash -vx cat > /tmp/test_csv.CSV <<EOF 1,"Tom",90,90,80 2,"Stanley",80,100,100 3,"Joseph",70,80,90 4,"Christy",80,80,100 5,"Chantelle",60,60,60 EOF mysql -uroot -p -e "create table test_csv ( no int not null, name varchar(32) not null, chi int not null, eng int not null, math int not null) engine=CSV;" world cp /tmp/test_csv.CSV /var/lib/mysql/world/test_csv.CSV mysql -uroot -p -e "flush table test_csv; select * from test_csv;" world Wish this helps. regards, Stanley Huang

[Level 2] Packing MyISAM data table into readonly.

/* step 1: LOCK TABLE x FOR WRITE; step 2: FLUSH TABLE x; step 3: myisamchk -cFU -- fast check for pre-existing errors. Don't pack if errors exist. Ignore the warning of table "not closed" because this check will go ahead and closed the DB files. step 4: myisampack -f -- force overwrite of any preexisting .TMD file step 5: myisamchk -raqS -- rebuild the index after pack step 6: FLUSH TABLE x; -- force reload of info_schema data step 7: UNLOCK TABLES; -- Release the table. */ mysql -u mysql <<EOF use test; lock table x for write; flush table x; EOF cd /opt/mysql/mysql/data/test; myisamchk -cFU x; mysiampack -f x; myisamchk -raqS x; mysql -u mysql <<EOF flush table x; unlock tables; EOF Wish this helps. regards, Stanley Huang

[Level 3] Multithread in Python

#!/bin/env python import os, sys import threading import random class ThreadClass (threading.Thread): def __init__(self, function, seconds, threads): self.function = function self.seconds = seconds threading.Thread.__init__(self) threads.append(self) def run(self): print 'run %s...' % self.seconds self.function(self.seconds) print 'run %s done!' % self.seconds class TestMultiThread(object): def __init__(self): self.max_thread = 5 self.threads = [] pass def task(self, seconds): os.popen('sleep %s' % seconds).read() def run(self): for i in range(0, self.max_thread): seconds = int(random.random() * 10) ThreadClass(self.task, seconds, self.threads).start() for t in self.threads: t.join() print 'Jobs done!' if __name__ == '__main__': tmt = TestMultiThread() tmt.ru

[Level 2] Flyweight in Python.

#!/usr/bin/env python import random class Black(object): __instance = None def __new__(cls): if not cls.__instance: cls.__instance = super(Black, cls).__new__(cls) return cls.__instance def __str__(self): return 'black' class White(object): __instance = None def __new__(cls): if not cls.__instance: cls.__instance = super(White, cls).__new__(cls) return cls.__instance def __str__(self): return 'white' class BlackAndWhite(object): __black = Black() __white = White() def __new__(cls, index): if index == 0: return cls.__black else: return cls.__white def __init__(self, index): pass class Board(object): def __init__(self, size): self.__size = size self.__planet = [ [ None for i in range(self.__size) ] for j in range(self.__size) ] def setPlanet(self): while True: x =