Here is an example of how to create the lustre file system from scratch. Note, ALL the systems MUST already be running the Luster kernel, with appropriate modules loaded. Once this is done, there is now a "lustre" file system on all machines, with kernel extensiosn to support it. The machine "miner" will be the MGS and MDT system, and machines n01 - n04 and miner will each be an OSS (Object Storage System) with one OST (Object Storage Target) each. 1) Create the MGS and MDS disk. ( a file system name 'spfs' with MGS and MDT on /dev/hde1 ) [root@miner nwhite]# PATH=${PATH}:/usr/sbin:/sbin [root@miner nwhite]# /usr/sbin/mkfs.lustre --fsname=spfs --reformat --mdt --mgs /dev/hde1 Permanent disk data: Target: spfs-MDTffff Index: unassigned Lustre FS: spfs Mount type: ldiskfs Flags: 0x75 (MDT MGS needs_index first_time update ) Persistent mount opts: errors=remount-ro,iopen_nopriv,user_xattr Parameters: device size = 47692MB formatting backing filesystem ldiskfs on /dev/hde1 target name spfs-MDTffff 4k blocks 0 options -J size=400 -i 4096 -I 512 -q -O dir_index -F mkfs_cmd = mkfs.ext2 -j -b 4096 -L spfs-MDTffff -J size=400 -i 4096 -I 512 -q -O dir_index -F /d ev/hde1 Writing CONFIGS/mountdata 2) Mount the MGS and MDS disk (this starts lustre, even though it has no OSS's or OST's yet) [root@miner nwhite]# mount -t lustre /dev/hde1 /lustre-mgs-mds [root@miner nwhite]# df Filesystem 1K-blocks Used Available Use% Mounted on /dev/hda3 239051908 19472848 209864556 9% / /dev/hda1 194442 139553 44850 76% /boot none 1037592 0 1037592 0% /dev/shm grid2:/gridwork 1032671136 709655744 312688704 70% /gridwork diskfarm:/disk2/sge 241263968 102308096 126700288 45% /sge halo:/vol/data2/is 545259520 480489408 64770112 89% /homedir/is rnd2:/rnddata 5284467136 4767905376 419924992 92% /mnt/rnddata leda2:/work/FC3-install 42465184 23463232 16844832 59% /ledawork/FC3-install /dev/hde1 42729112 463188 39824048 2% /lustre-mgs-mds [root@miner nwhite]# more /etc/fstab # This file is edited by fstab-sync - see 'man fstab-sync' for details /dev/hda3 / ext3 defaults 1 1 LABEL=/boot /boot ext3 defaults 1 2 none /dev/pts devpts gid=5,mode=620 0 0 none /dev/shm tmpfs defaults 0 0 none /proc proc defaults 0 0 none /sys sysfs defaults 0 0 /dev/hda2 swap swap defaults 0 0 grid2:/gridwork /gridwork nfs grid2:/griddata /griddata nfs diskfarm:/disk2/sge /sge nfs #/dev/hde1 /minerwork ext3 defaults 1 2 #/minerwork/swapfile swap swap defaults 0 0 /dev/hde1 /lustre-mgs-mds lustre defaults,_netdev 0 0 /dev/hde2 /lustre-ost lustre defaults,_netdev 0 0 # apparently, should run client and ost on same machine #192.168.0.100@tcp0:/spfs /lustrework lustre defaults 0 0 /dev/hdc /media/cdrom auto pamconsole,exec,noauto,managed 0 0 /dev/fd0 /media/floppy auto pamconsole,exec,noauto,managed 0 0 3) Check to see if things are actually loaded.. [root@miner nwhite]# cat /proc/fs/lustre/devices 0 UP mgs MGS MGS 5 1 UP mgc MGC192.168.0.100@tcp 4b5270c2-1256-7526-b63f-f4da3b82ea22 5 2 UP mdt MDS MDS_uuid 3 3 UP lov spfs-mdtlov spfs-mdtlov_UUID 4 4 UP mds spfs-MDT0000 spfs-MDT0000_UUID 3 4) Create on OST on machine miner... on /dev/hde2 [root@miner nwhite]# /usr/sbin/mkfs.lustre --fsname=spfs --reformat --ost --mgsnode=192.168.0.100@tcp0 /dev/hde2 Permanent disk data: Target: spfs-OSTffff Index: unassigned Lustre FS: spfs Mount type: ldiskfs Flags: 0x72 (OST needs_index first_time update ) Persistent mount opts: errors=remount-ro,extents,mballoc Parameters: mgsnode=192.168.0.100@tcp device size = 238488MB formatting backing filesystem ldiskfs on /dev/hde2 target name spfs-OSTffff 4k blocks 0 options -J size=400 -i 16384 -I 256 -q -O dir_index -F mkfs_cmd = mkfs.ext2 -j -b 4096 -L spfs-OSTffff -J size=400 -i 16384 -I 256 -q -O dir_index -F /dev/hde2 Writing CONFIGS/mountdata [root@miner nwhite]# more /etc/fstab # This file is edited by fstab-sync - see 'man fstab-sync' for details /dev/hda3 / ext3 defaults 1 1 LABEL=/boot /boot ext3 defaults 1 2 none /dev/pts devpts gid=5,mode=620 0 0 none /dev/shm tmpfs defaults 0 0 none /proc proc defaults 0 0 none /sys sysfs defaults 0 0 /dev/hda2 swap swap defaults 0 0 grid2:/gridwork /gridwork nfs grid2:/griddata /griddata nfs diskfarm:/disk2/sge /sge nfs #/dev/hde1 /minerwork ext3 defaults 1 2 #/minerwork/swapfile swap swap defaults 0 0 /dev/hde1 /lustre-mgs-mds lustre defaults,_netdev 0 0 /dev/hde2 /lustre-ost lustre defaults,_netdev 0 0 # apparently, should run client and ost on same machine #192.168.0.100@tcp0:/spfs /lustrework lustre defaults 0 0 /dev/hdc /media/cdrom auto pamconsole,exec,noauto,managed 0 0 /dev/fd0 /media/floppy auto pamconsole,exec,noauto,managed 0 0 MOUNT THE OST (this should be in /etc/fstab eventually) [root@miner nwhite]# mount -t lustre /dev/hde2 /lustre-ost NOW GO TO OTHER SYSTEMS and CREATE OSTs [root@miner nwhite]# ssh n03 Last login: Sat Dec 22 12:32:52 2007 from miner.stern.nyu.edu [root@n03 ~]# more /etc/fstab # This file is edited by fstab-sync - see 'man fstab-sync' for details LABEL=/1 / ext3 defaults 1 1 LABEL=/boot /boot ext3 defaults 1 2 none /dev/pts devpts gid=5,mode=620 0 0 none /dev/shm tmpfs defaults 0 0 /dev/hdc /lustre-ost lustre defaults,_netdev 0 0 # 192.168.0.100@tcp:/lustre /lustrework lustre defaults 0 0 none /proc proc defaults 0 0 none /sys sysfs defaults 0 0 LABEL=/var /var ext3 defaults 1 2 LABEL=SWAP-hda5 swap swap defaults 0 0 /dev/fd0 /media/floppy auto pamconsole,exec,noauto,managed 0 0 [root@n03 ~]# /usr/sbin/mkfs.lustre --fsname=spfs --reformat --ost --mgsnode=192.168.0.100@tcp0 /dev/hdc Permanent disk data: Target: spfs-OSTffff Index: unassigned Lustre FS: spfs Mount type: ldiskfs Flags: 0x72 (OST needs_index first_time update ) Persistent mount opts: errors=remount-ro,extents,mballoc Parameters: mgsnode=192.168.0.100@tcp device size = 305245MB formatting backing filesystem ldiskfs on /dev/hdc target name spfs-OSTffff 4k blocks 0 options -J size=400 -i 16384 -I 256 -q -O dir_index -F mkfs_cmd = mkfs.ext2 -j -b 4096 -L spfs-OSTffff -J size=400 -i 16384 -I 256 -q -O dir_index -F /dev/hdc Writing CONFIGS/mountdata [root@n03 ~]# more /etc/fstab # This file is edited by fstab-sync - see 'man fstab-sync' for details LABEL=/1 / ext3 defaults 1 1 LABEL=/boot /boot ext3 defaults 1 2 none /dev/pts devpts gid=5,mode=620 0 0 none /dev/shm tmpfs defaults 0 0 /dev/hdc /lustre-ost lustre defaults,_netdev 0 0 # 192.168.0.100@tcp:/lustre /lustrework lustre defaults 0 0 none /proc proc defaults 0 0 none /sys sysfs defaults 0 0 LABEL=/var /var ext3 defaults 1 2 LABEL=SWAP-hda5 swap swap defaults 0 0 /dev/fd0 /media/floppy auto pamconsole,exec,noauto,managed 0 0 CREATE MOUNT POINT [root@n03 ~]# mkdir -p /lustre-ost MOUNT OST (At this point it's space is available to cluster file system [root@n03 ~]# mount -t lustre /dev/hdc /lustre-ost REPEAT for n04... Cluster file system should now be ready, Try to connect a client to it. SSH to EUCLID ( a client machine) create a mount point for the file system at /lustrework mkdir -p /lustrework Add the following line to /etc/fstab 192.168.0.100@tcp0:/spfs /lustrework lustre defaults 0 0 Then mount the file system mount /lustrework See if it is there.. [root@euclid ~]# df -h Filesystem Size Used Avail Use% Mounted on /dev/sda2 95G 4.4G 86G 5% / /dev/sda1 99M 70M 25M 74% /boot none 8.0G 0 8.0G 0% /dev/shm /dev/sda5 16G 79M 15G 1% /tmp /dev/sda6 7.9G 2.0G 5.5G 27% /var diskfarm:/usr/install 224G 44G 169G 21% /usr/install diskfarm:/disk2/sge 231G 98G 121G 45% /sge 192.168.0.100@tcp:/spfs 817G 1.4G 774G 1% /lustrework [root@euclid ~]# ALL Done on euclid Repeat on darwin..