Common text processing tools

Keywords: Nginx

paste -d"#" Merges files horizontally, with a '?' as the separator
cut -d '%' -f2,3-5 Take the second column and three to five columns with the% separator
cat f1 f2 Merge files vertically
tac f1 f2 It's the opposite of cat
tr -s " " "%" Space compressed to%
tr -s " " Compress spaces only
[root@nginx ~]#  ifconfig eth0
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.186.39.193  netmask 255.255.0.0  broadcast 10.186.255.255
        ether 52:54:00:33:78:f8  txqueuelen 1000  (Ethernet)
        RX packets 8942034  bytes 5666332700 (5.2 GiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 7219762  bytes 1850180335 (1.7 GiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@nginx ~]#  ifconfig eth0 |head -n2 |tail -n1
        inet 10.186.39.193  netmask 255.255.0.0  broadcast 10.186.255.255
[root@nginx ~]#  ifconfig eth0 |head -n2 |tail -n1 |tr -s ' ' | cut -d ' '  -f3
10.186.39.193

Sort default sort
-Tmake separa t or
-u de reorder
-n order (from small to large)
-r is opposite to - n
-R random order (generate random number, such as lottery)
-k specify which column

[root@nginx ~]# echo {1..50} | tr -s " " "\n" |sort -R |head -n1  #Generate random number
12
[root@nginx ~]# echo {1..50} | tr -s " " "\n" |sort -R |head -n1
46
[root@nginx ~]# cat /var/log/nginx/access.log | cut -d' ' -f1 | sort -u -n   #Sort and de duplicate
1.202.69.104
5.8.54.27
23.254.203.250
36.152.17.61
37.59.57.78
43.226.144.199
52.222.3.206
52.77.223.212

uniq
-c shows the number of repetitions
-d show only duplicate lines
-u only show lines that have not been repeated

[root@nginx ~]#  cat /var/log/nginx/access.log | cut -d " " -f1 |uniq -c |sort -nr |head -n10   #Top 10 IP rankings for a website visit
    400 119.29.143.122
    252 43.226.144.199
     13 222.128.2.100
     13 1.202.69.104
     12 82.223.29.247
     12 37.59.57.78
     12 188.40.127.8
     12 120.55.164.49
     10 52.222.3.206
      4 59.110.237.76
[root@nginx ~]# sort -t " " -k1  /var/log/nginx/access.log | cut -d " " -f1 |uniq -c |sort -nr |head -n10   #It's also the top ten IP sorting for access
[root@nginx ~]# sort -t " " -k1  /var/log/nginx/access.log | cut -d " " -f1 |uniq -c |sort -nr |head -n10 |tr -s " " |cut -d " "  -f3  #Number of times before removal
119.29.143.122
43.226.144.199
222.128.2.100
1.202.69.104
82.223.29.247
52.222.3.206
37.59.57.78
188.40.127.8
120.55.164.49
59.110.237.76
[root@nginx ~]# cat f2 
aa
bb
deklfmef
ferf
er
[root@nginx ~]# cat f1 
aa
bb
ddddd
cc
ggg
[root@nginx ~]#  cat f1 f2 | sort |uniq -d  #Find the same line in two files
aa
bb

Posted by intellivision on Sun, 22 Dec 2019 14:10:06 -0800