Unix Tips

Here's a jumble of my notes on Unix utilities

  1. Awk, nawk, sed, and regular expressions
    right justify 1st field:
    cat bigdir.dat | grep mitch | awk '{printf "%10d %s\n",$1,$2}' | sort > t2
    right justify 1st field:
    cat filename | awk '{printf "%10d %s\n",$1,$2}'
    sum numbers in first column of a file
    cat filename | awk 'BEGIN { tot=0 } { tot+=$1 } END { print tot }'
    cat wellprod.list | sed -e 's/\([1-3]\)\([ ]*\)/\1/'  // to replace all blanks after a 1,2, or 3 with just the 1,2,3
    grep handle\-name *.G | nawk '{ print $3 }' | sed 's/"//g' | en  //enscript all handle-names
    grep composition *.G | grep -v order | nawk '{ print $3 " " $4 " " $5 }' | sed 's/"//g' | en to see functions
    lpq | nawk '{ printf("%d ", $3)}'       //print sans lf
    
    print only the second column from file t1
    cat t1 | awk '{print $2}'
  2. sed
    only print certain lines in a file, eq 20 - 30
    sed -n -e 20,30p filename
  3. tail
    remove the first line from a file
    tail +1 filename
  4. Searching, with find and egrep.
    Search all files in and below for a string:
    find . -exec egrep string {} \; -print
    Search all files in and below for a string and not see errors:
    find . -exec egrep string {} \; -print 2> /dev/null
    Search all files with names matching "*.shtml" and replace "http" with "https"
    find -type f -name "*.shtml" -exec sed -i "s/http:\/\/www.fincher.org/https:\/\/www.fincher.org/g" {} \;
    Find and print all files in this directory and subdirectories with certain extensions
    find . -type f \( -name "*.JPEG" -o -name "*.jpeg" \)
    Read a list of search strings, one per line, from the file checksum.tmp and search for those strings in file checksumAndFilename.tmp
    grep -f checksum.tmp checksumAndFilename.tmp
    Remove all files named ".depend.make" from here and below
    find . -name .depend.make -exec rm {} \;
    Remove all files name '*.JRN.*' from here and below
    find . -name '*.JRN.*' -exec rm {} \; -print
    ls all files name '*.JRN.*' or "core" from here and below
    find . \( -name '*.JRN.*' -o -name 'core' \) -exec ls -la {} \; -print
    search a file for any of three strings:
    egrep 'this|that|the_other' filename
    delay start of a shell script:
    at -s now + 8 hours sh-script 2>> /dev/null
    delay start of a shell script:
    at -s 1 am tomorrow sh-script 2>> /dev/null
    list all files below current dir:
    du -a .
    example of remote copy:
    rcp -r john@zoom:/pa/tcltest .
    rcp mitch@holyman:/export/home8/dino/mitch/SeismicObjects.ps .
    list files in reverse time order:
    ls -lst
    /usr/ucb/fastboot
  5. Miscellaneous
    Print only the lines that are duplicates in the sorted file t1
    uniq -D t1
    How to set up cron jobs (see man crontab)
    crontab < .crontab
    How to see cron jobs
    crontab -l
    How do I remove all the spaces/blanks in a file?
    cat filename | tr -d ' '
    How to tell versions on some executables
    strings executable_name | egrep -i version
    How to search recursively all directories for duplicate images
    find . -type f \( -name "*.jpg" -o -name "*.gif" \) | awk '{print "\"" $0 "\""}' | xargs shasum -a 256 | sort > checksumAndFilename.tmp && cat checksumAndFilename.tmp | awk '{print $1}' | uniq -D > checksum.tmp && grep -f checksum.tmp checksumAndFilename.tmp | tee duplicates.tmp && echo "output in \"duplicates.tmp\"" && rm checksumAndFilename.tmp checksum.tmp
  6. tar - working with tapes and such
    tar files into a file:
    tar cvf strat.tar *
    untar a floppy:
    tar -xvf /dev/fd0; eject /dev/fd0
    move ~/pman to /misc/mitch/pman
    cd ~;tar clf - pman | ( cd /misc/mitch ; tar xBevplf - )
    to download
    mt -f /dev/rst0 rew ; tar dvf /dev/rst0 dir_name;
    load a tape into current directory
    tar xvf /dev/rst0
    ls a tape
    tar tvf /dev/rst0
    tar -xvf tarfilename.tar
    extract everything, otherwise you can use:
    tar -tvf tarfilename.tar
    view the contents
    tar -xvf tarfilename.tar filename
    create save tape
    tar cvlf /dev/rst0 objcenter
  7. Bourne shell redirection
    0 is stdin; 1 is stdout; 2 is stderr
    redirect std err to file
    command 2>file
    direct the output from command to stderr
    commamd >& 2
    direct std output and std err to foo or /dev/null
    command >foo 2>>foo
    same as above stderr rerouted to whatever filedes 1 is.
    command >foo 2>&1
    if [ -z "$FRED" ]; then  //// to test for an empty string
        echo FRED not set
    fi
       test -n "$FRED"  // to test for an existing string
    
  8. C shell
    ( find . -name .cshrc -print < tempfile) > & /dev/null    //redirect stderr
    csh: t.sh >& t5  //redirect both stdout and stderr to file
    
    set path= ( $path /home/ow31b/class2 ) //add 1 dir to path in csh
    To substitue the second all instances of something
    % cp fred1.txt fred01.txt 
    % !!:gs/1/2 cp fred2.txt fred02.txt //global command line substitution
    
    How to branch on whether a command is successful?
    echo $host | grep x2s
    if [ $? -eq 0 ]
    then
       echo I found it
    else
      echo Nothing found
    fi
    
  9. ksh:
      .spacefilename is same as "source filename"
      esc k : to back up commands
      
  10. imake
    /usr/openwin/lib/config   /imake rules and stuff
    imake -DUseInstalled -I/usr/local/src/config
    xmkmf;mv Makefile Makefile.bak;imake -DUseInstalled /usr/openwin/lib/config
    
  11. misc
    genealogy:~$ telnet genealogy.org 2000  //get soundex info
    nenscript -B -fCourier10 wl
    daVinci -startappl grapheditor grapheditor
    swap -s // to find the amount of swap space on an sgi IRIX 5.3.
    % man command | col -b > command.man // to asciize man file
    % basename dirname // grabs filename from variable
    % /etc/services tells socket connections
    more /etc/vfstab   mounting stuff
    
    
  12. Vi
    redirect std err to file
    command 2>file
    to substitue all occurrances of a string
    %s/oldstring/newstring/g
    to write a buffer to another fi
    :w filename
    to write lines of a buffer to another fi
    :100,120w filename
  13. tcsh
    echo $tcsh  // shows version
    set autolist // upon receiving a ^d this prints options
     echo $shell //tell what shell I am using
    % mv chapter{8,9} is equivalent to % mv chapter8 chapter9
    % cp .cshrc{,.bak}  % cp .cshrc .cshrc.bak
    
  14. ksh print $((RANDOM%14+1)) //print a random number To update the list of topics in info, edit info/dir and add a line.
  15. Make A sample make file to recurse thru subdirectories (I know all the cool people use 'Ant', but for historical reasons...)
    SUBDIRS=questions databases servlets util
    
    all:  compile
    
    compile: 
    	for DIR in $(SUBDIRS) ; do \
    	( echo "**** invoking \"$@\" on subdirectory $$DIR ******" ; cd $$DIR ; $(MAKE) ) ; done
    
    clean: 
    	for DIR in $(SUBDIRS) ; do \
    	( echo "**** invoking $@ on subdirectory $$DIR ******" ; cd $$DIR ; $(MAKE) $@) ; done
    
    a sample master.mak
    JAVADIR=W:/Projects/tools/i386/win32/sun/java/1.1.7
    JAVABINDIR=$(JAVADIR)/bin
    JAVAC = $(JAVABINDIR)/javac 
    JAVA = $(JAVABINDIR)/java
    SERVLETS=W:/Projects/src/surveynet/builder/servlets/ViewResults;W:/Projects/rel/debug/generic/java/javalib;W:/Projects/rel/common/generic/java/lib
    CLASSPATH=.;W:/Projects/rel/common/generic/java/lib/sun/xml/xml.jar;$(JAVADIR)/lib;$(JAVADIR)/jre/lib;$(JAVADIR)/lib/classes.zip;$(SERVLETS);../../../../
    
    JAVAOPTS=  -Djava.compiler=NONE
    JAVACOPTS= 
    
    
    %.class : %.java
    	$(JAVAC) -classpath $(subst ;,\;, $(CLASSPATH)) $(JAVACOPTS) $<
    
    A sample make file to build a directory
    include $(HOPLITEROOT)/master.mak
    
    JAVAFILES=SurveyDatabase SurveyDatabaseH3 ConnectionPool
    
    all:  $(foreach JF, $(JAVAFILES), $(JF).class)
    
    clean:
    	-rm *.class
    
    Another sample make file for working only with one file
    include master.mak
    
    JAVAFILE=GraphicEnv
    
    all:  $(JAVAFILE).class run
    
    run: 
    	@echo "$@..."
    	$(JAVA) -classpath $(CLASSPATH) $(JAVAOPTS) $(JAVAFILE)
    
    clean:
    	-rm $(JAVAFILE).class
    

"Unix is user-friendly. It's just very selective about who its friends are."