Here is an example of how to create the lustre file system from scratch. 

Note, ALL the systems MUST already be running the Luster kernel, with appropriate modules loaded. Once this is done, there is now a "lustre" file system on all machines, with kernel extensiosn to support it.




The machine "miner" will be the MGS and MDT system, and machines n01 - n04 and miner will each be an OSS (Object Storage System) with one OST (Object Storage Target) each.




1) Create the MGS and MDS disk. ( a file system name 'spfs'   with MGS and MDT on /dev/hde1 )


[root@miner nwhite]# PATH=${PATH}:/usr/sbin:/sbin
[root@miner nwhite]# /usr/sbin/mkfs.lustre --fsname=spfs --reformat --mdt --mgs /dev/hde1

   Permanent disk data:
Target:     spfs-MDTffff
Index:      unassigned
Lustre FS:  spfs
Mount type: ldiskfs
Flags:      0x75
              (MDT MGS needs_index first_time update )
Persistent mount opts: errors=remount-ro,iopen_nopriv,user_xattr
Parameters:

device size = 47692MB
formatting backing filesystem ldiskfs on /dev/hde1
        target name  spfs-MDTffff
        4k blocks     0
        options        -J size=400 -i 4096 -I 512 -q -O dir_index -F
mkfs_cmd = mkfs.ext2 -j -b 4096 -L spfs-MDTffff  -J size=400 -i 4096 -I 512 -q -O dir_index -F /d
ev/hde1
Writing CONFIGS/mountdata


2) Mount the MGS and MDS disk (this starts lustre, even though it has no OSS's or OST's yet)

[root@miner nwhite]# mount -t lustre /dev/hde1 /lustre-mgs-mds
[root@miner nwhite]# df

Filesystem           1K-blocks      Used Available Use% Mounted on
/dev/hda3            239051908  19472848 209864556   9% /
/dev/hda1               194442    139553     44850  76% /boot
none                   1037592         0   1037592   0% /dev/shm
grid2:/gridwork      1032671136 709655744 312688704  70% /gridwork
diskfarm:/disk2/sge  241263968 102308096 126700288  45% /sge
halo:/vol/data2/is   545259520 480489408  64770112  89% /homedir/is
rnd2:/rnddata        5284467136 4767905376 419924992  92% /mnt/rnddata
leda2:/work/FC3-install
                      42465184  23463232  16844832  59% /ledawork/FC3-install
/dev/hde1             42729112    463188  39824048   2% /lustre-mgs-mds


[root@miner nwhite]# more /etc/fstab
# This file is edited by fstab-sync - see 'man fstab-sync' for details
/dev/hda3                /                       ext3    defaults        1 1
LABEL=/boot             /boot                   ext3    defaults        1 2
none                    /dev/pts                devpts  gid=5,mode=620  0 0
none                    /dev/shm                tmpfs   defaults        0 0
none                    /proc                   proc    defaults        0 0
none                    /sys                    sysfs   defaults        0 0
/dev/hda2               swap                    swap    defaults        0 0
grid2:/gridwork /gridwork nfs
grid2:/griddata /griddata nfs
diskfarm:/disk2/sge /sge nfs
#/dev/hde1        /minerwork               ext3     defaults   1 2 
#/minerwork/swapfile       swap            swap     defaults   0 0 
/dev/hde1               /lustre-mgs-mds         lustre  defaults,_netdev        0 0
/dev/hde2               /lustre-ost             lustre  defaults,_netdev        0 0
#  apparently, should run client and ost on same machine
#192.168.0.100@tcp0:/spfs       /lustrework     lustre  defaults        0 0
/dev/hdc                /media/cdrom            auto    pamconsole,exec,noauto,managed 0 0
/dev/fd0                /media/floppy           auto    pamconsole,exec,noauto,managed 0 0

3) Check to see if things are actually loaded..

[root@miner nwhite]# cat /proc/fs/lustre/devices
  0 UP mgs MGS MGS 5
  1 UP mgc MGC192.168.0.100@tcp 4b5270c2-1256-7526-b63f-f4da3b82ea22 5
  2 UP mdt MDS MDS_uuid 3
  3 UP lov spfs-mdtlov spfs-mdtlov_UUID 4
  4 UP mds spfs-MDT0000 spfs-MDT0000_UUID 3

4) Create on OST on machine miner... on /dev/hde2


[root@miner nwhite]# /usr/sbin/mkfs.lustre --fsname=spfs --reformat --ost --mgsnode=192.168.0.100@tcp0  /dev/hde2

   Permanent disk data:
Target:     spfs-OSTffff
Index:      unassigned
Lustre FS:  spfs
Mount type: ldiskfs
Flags:      0x72
              (OST needs_index first_time update )
Persistent mount opts: errors=remount-ro,extents,mballoc
Parameters: mgsnode=192.168.0.100@tcp

device size = 238488MB
formatting backing filesystem ldiskfs on /dev/hde2
        target name  spfs-OSTffff
        4k blocks     0
        options        -J size=400 -i 16384 -I 256 -q -O dir_index -F
mkfs_cmd = mkfs.ext2 -j -b 4096 -L spfs-OSTffff  -J size=400 -i 16384 -I 256 -q -O dir_index -F /dev/hde2
Writing CONFIGS/mountdata


[root@miner nwhite]# more /etc/fstab
# This file is edited by fstab-sync - see 'man fstab-sync' for details
/dev/hda3                /                       ext3    defaults        1 1
LABEL=/boot             /boot                   ext3    defaults        1 2
none                    /dev/pts                devpts  gid=5,mode=620  0 0
none                    /dev/shm                tmpfs   defaults        0 0
none                    /proc                   proc    defaults        0 0
none                    /sys                    sysfs   defaults        0 0
/dev/hda2               swap                    swap    defaults        0 0
grid2:/gridwork /gridwork nfs
grid2:/griddata /griddata nfs
diskfarm:/disk2/sge /sge nfs
#/dev/hde1        /minerwork               ext3     defaults   1 2 
#/minerwork/swapfile       swap            swap     defaults   0 0 
/dev/hde1               /lustre-mgs-mds         lustre  defaults,_netdev        0 0
/dev/hde2               /lustre-ost             lustre  defaults,_netdev        0 0
#  apparently, should run client and ost on same machine
#192.168.0.100@tcp0:/spfs       /lustrework     lustre  defaults        0 0
/dev/hdc                /media/cdrom            auto    pamconsole,exec,noauto,managed 0 0
/dev/fd0                /media/floppy           auto    pamconsole,exec,noauto,managed 0 0

MOUNT THE OST   (this should be in /etc/fstab eventually)

[root@miner nwhite]# mount -t lustre /dev/hde2 /lustre-ost

NOW GO TO OTHER SYSTEMS and CREATE OSTs

[root@miner nwhite]# ssh n03
Last login: Sat Dec 22 12:32:52 2007 from miner.stern.nyu.edu
[root@n03 ~]# more /etc/fstab
# This file is edited by fstab-sync - see 'man fstab-sync' for details
LABEL=/1                /                       ext3    defaults        1 1
LABEL=/boot             /boot                   ext3    defaults        1 2
none                    /dev/pts                devpts  gid=5,mode=620  0 0
none                    /dev/shm                tmpfs   defaults        0 0
/dev/hdc                /lustre-ost             lustre  defaults,_netdev        0 0
# 192.168.0.100@tcp:/lustre     /lustrework     lustre  defaults        0 0
none                    /proc                   proc    defaults        0 0
none                    /sys                    sysfs   defaults        0 0
LABEL=/var              /var                    ext3    defaults        1 2
LABEL=SWAP-hda5         swap                    swap    defaults        0 0
/dev/fd0                /media/floppy           auto    pamconsole,exec,noauto,managed 0 0




[root@n03 ~]#  /usr/sbin/mkfs.lustre --fsname=spfs --reformat --ost --mgsnode=192.168.0.100@tcp0  /dev/hdc

   Permanent disk data:
Target:     spfs-OSTffff
Index:      unassigned
Lustre FS:  spfs
Mount type: ldiskfs
Flags:      0x72
              (OST needs_index first_time update )
Persistent mount opts: errors=remount-ro,extents,mballoc
Parameters: mgsnode=192.168.0.100@tcp

device size = 305245MB
formatting backing filesystem ldiskfs on /dev/hdc
        target name  spfs-OSTffff
        4k blocks     0
        options        -J size=400 -i 16384 -I 256 -q -O dir_index -F
mkfs_cmd = mkfs.ext2 -j -b 4096 -L spfs-OSTffff  -J size=400 -i 16384 -I 256 -q -O dir_index -F /dev/hdc
Writing CONFIGS/mountdata





[root@n03 ~]# more /etc/fstab
# This file is edited by fstab-sync - see 'man fstab-sync' for details
LABEL=/1                /                       ext3    defaults        1 1
LABEL=/boot             /boot                   ext3    defaults        1 2
none                    /dev/pts                devpts  gid=5,mode=620  0 0
none                    /dev/shm                tmpfs   defaults        0 0
/dev/hdc                /lustre-ost             lustre  defaults,_netdev        0 0
# 192.168.0.100@tcp:/lustre     /lustrework     lustre  defaults        0 0
none                    /proc                   proc    defaults        0 0
none                    /sys                    sysfs   defaults        0 0
LABEL=/var              /var                    ext3    defaults        1 2
LABEL=SWAP-hda5         swap                    swap    defaults        0 0
/dev/fd0                /media/floppy           auto    pamconsole,exec,noauto,managed 0 0

CREATE MOUNT POINT

[root@n03 ~]# mkdir -p /lustre-ost

MOUNT OST   (At this point it's space is available to cluster file system

[root@n03 ~]# mount -t lustre /dev/hdc /lustre-ost


REPEAT for n04...




Cluster file system should now be ready, Try to connect a client to it. 

SSH to EUCLID ( a client machine)


 create a mount point for the file system at /lustrework

mkdir -p /lustrework



Add the following line to /etc/fstab

192.168.0.100@tcp0:/spfs     /lustrework     lustre  defaults        0 0


Then  mount the file system

mount /lustrework


See if it is there..

[root@euclid ~]# df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/sda2              95G  4.4G   86G   5% /
/dev/sda1              99M   70M   25M  74% /boot
none                  8.0G     0  8.0G   0% /dev/shm
/dev/sda5              16G   79M   15G   1% /tmp
/dev/sda6             7.9G  2.0G  5.5G  27% /var
diskfarm:/usr/install
                      224G   44G  169G  21% /usr/install
diskfarm:/disk2/sge   231G   98G  121G  45% /sge
192.168.0.100@tcp:/spfs
                      817G  1.4G  774G   1% /lustrework
[root@euclid ~]# 


ALL Done on euclid

Repeat on darwin..