From d26560950b6ba6454c11cd978d3e6bb4d38430e8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 9 May 2018 10:18:49 -0300 Subject: scripts/documentation-file-ref-check: rewrite it in perl with auto-fix mode The original shell script works, but: 1) it is too slow; 2) it is hard to exclude rejex patterns Convert it to perl. Here, the new version is able to check the entire tree in less than a second (after cached): real 0m0,284s user 0m0,668s sys 0m0,778s The old version takes more than a minute to complete (also after cached): real 1m17,905s user 0m25,583s sys 0m55,334s It also produce less false-positives (if any). The new script also contains an auto-fix mode. Usually, file references get lost when they're moved to some other place and/or renamed to .rst. Add an experimental mode to auto-fix those. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- scripts/documentation-file-ref-check | 125 +++++++++++++++++++++++++++++++---- 1 file changed, 113 insertions(+), 12 deletions(-) (limited to 'scripts/documentation-file-ref-check') diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index bc1659900e891..2520bc14ffacf 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -1,15 +1,116 @@ -#!/bin/sh +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 +# # Treewide grep for references to files under Documentation, and report # non-existing files in stderr. -for f in $(git ls-files); do - for ref in $(grep -ho "Documentation/[A-Za-z0-9_.,~/*+-]*" "$f"); do - # presume trailing . and , are not part of the name - ref=${ref%%[.,]} - - # use ls to handle wildcards - if ! ls $ref >/dev/null 2>&1; then - echo "$f: $ref" >&2 - fi - done -done +use warnings; +use strict; +use Getopt::Long qw(:config no_auto_abbrev); + +my $scriptname = $0; +$scriptname =~ s,.*/([^/]+/),$1,; + +# Parse arguments +my $help = 0; +my $fix = 0; + +GetOptions( + 'fix' => \$fix, + 'h|help|usage' => \$help, +); + +if ($help != 0) { + print "$scriptname [--help] [--fix-rst]\n"; + exit -1; +} + +# Step 1: find broken references +print "Finding broken references. This may take a while... " if ($fix); + +my %broken_ref; + +open IN, "git grep 'Documentation/'|" + or die "Failed to run git grep"; +while () { + next if (!m/^([^:]+):(.*)/); + + my $f = $1; + my $ln = $2; + + # Makefiles contain nasty expressions to parse docs + next if ($f =~ m/Makefile/); + # Skip this script + next if ($f eq $scriptname); + + if ($ln =~ m,\b(\S*)(Documentation/[A-Za-z0-9\_\.\,\~/\*+-]*),) { + my $prefix = $1; + my $ref = $2; + my $base = $2; + + $ref =~ s/[\,\.]+$//; + + my $fulref = "$prefix$ref"; + + $fulref =~ s/^(\ 1) { + print STDERR "WARNING: Won't auto-replace, as found multiple files close to $ref:\n"; + foreach my $j (@find) { + $j =~ s,^./,,; + print STDERR " $j\n"; + } + } else { + $f = $find[0]; + $f =~ s,^./,,; + print "INFO: Replacing $ref to $f\n"; + foreach my $j (qx(git grep -l $ref)) { + qx(sed "s\@$ref\@$f\@g" -i $j); + } + } +} -- cgit v1.2.3 From 40fc3eb055cb39429dc16a94e825a5598befb9a7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 14 Jun 2018 07:11:02 -0300 Subject: scripts/documentation-file-ref-check: fix help message The name of the --fix option was renamed, but it was not changed at the quick help message. Signed-off-by: Mauro Carvalho Chehab Acked-by: Jonathan Corbet --- scripts/documentation-file-ref-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/documentation-file-ref-check') diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index 2520bc14ffacf..9e7ae14080aa7 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -21,7 +21,7 @@ GetOptions( ); if ($help != 0) { - print "$scriptname [--help] [--fix-rst]\n"; + print "$scriptname [--help] [--fix]\n"; exit -1; } -- cgit v1.2.3 From 50440240021ca3cefe376fa11a7ae085734f165a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 14 Jun 2018 07:48:22 -0300 Subject: scripts/documentation-file-ref-check: accept more wildcards at filenames at MAINTAINERS, some filename paths use '?' and things like [7,9]. So, accept more wildcards, in order to avoid false-positives. Signed-off-by: Mauro Carvalho Chehab Acked-by: Jonathan Corbet --- scripts/documentation-file-ref-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/documentation-file-ref-check') diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index 9e7ae14080aa7..9d5e21b10346b 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -43,7 +43,7 @@ while () { # Skip this script next if ($f eq $scriptname); - if ($ln =~ m,\b(\S*)(Documentation/[A-Za-z0-9\_\.\,\~/\*+-]*),) { + if ($ln =~ m,\b(\S*)(Documentation/[A-Za-z0-9\_\.\,\~/\*\[\]\?+-]*),) { my $prefix = $1; my $ref = $2; my $base = $2; -- cgit v1.2.3 From be600e5ac79fb5f6c8839d6a48d89fc2b917f0c7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 14 Jun 2018 09:36:35 -0300 Subject: scripts/documentation-file-ref-check: add a fix logic for DT There are several links broken due to DT file movements. Add a hint logic to seek for those changes. Signed-off-by: Mauro Carvalho Chehab Acked-by: Jonathan Corbet --- scripts/documentation-file-ref-check | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'scripts/documentation-file-ref-check') diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index 9d5e21b10346b..c8bc1c1c1d6e2 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -64,7 +64,7 @@ while () { next if (grep -e, glob("$ref $fulref")); if ($fix) { - if (!($ref =~ m/(devicetree|scripts|Kconfig|Kbuild)/)) { + if (!($ref =~ m/(scripts|Kconfig|Kbuild)/)) { $broken_ref{$ref}++; } } else { @@ -84,10 +84,19 @@ foreach my $ref (keys %broken_ref) { # get just the basename $new =~ s,.*/,,; - # Seek for the same name on another place, as it may have been moved my $f=""; - $f = qx(find . -iname $new) if ($new); + # usual reason for breakage: DT file moved around + if ($ref =~ /devicetree/) { + my $search = $new; + $search =~ s,^.*/,,; + $f = qx(find Documentation/devicetree/ -iname "*$search*") if ($search); + if (!$f) { + # Manufacturer name may have changed + $search =~ s/^.*,//; + $f = qx(find Documentation/devicetree/ -iname "*$search*") if ($search); + } + } # usual reason for breakage: file renamed to .rst if (!$f) { @@ -95,6 +104,11 @@ foreach my $ref (keys %broken_ref) { $f=qx(find . -iname $new) if ($new); } + # Wild guess: seek for the same name on another place + if (!$f) { + $f = qx(find . -iname $new) if ($new); + } + my @find = split /\s+/, $f; if (!$f) { -- cgit v1.2.3 From e1f319fe4d537da79691f1a1da7a20147de33047 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 14 Jun 2018 10:14:54 -0300 Subject: scripts/documentation-file-ref-check: hint: dash or underline Sometimes, people use dash instead of underline or vice-versa. Try to autocorrect it. Signed-off-by: Mauro Carvalho Chehab Acked-by: Jonathan Corbet --- scripts/documentation-file-ref-check | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'scripts/documentation-file-ref-check') diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index c8bc1c1c1d6e2..d132f756d31d4 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -104,6 +104,12 @@ foreach my $ref (keys %broken_ref) { $f=qx(find . -iname $new) if ($new); } + # usual reason for breakage: use dash or underline + if (!$f) { + $new =~ s/[-_]/[-_]/g; + $f=qx(find . -iname $new) if ($new); + } + # Wild guess: seek for the same name on another place if (!$f) { $f = qx(find . -iname $new) if ($new); -- cgit v1.2.3 From 2d69708f9c08067735672908507894374bebb379 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 14 Jun 2018 10:47:29 -0300 Subject: scripts/documentation-file-ref-check: get rid of false-positives Now that the number of broken refs are smaller, improve the logic that gets rid of false-positives. Signed-off-by: Mauro Carvalho Chehab Acked-by: Jonathan Corbet --- scripts/documentation-file-ref-check | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'scripts/documentation-file-ref-check') diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index d132f756d31d4..047f463cdf4be 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -38,16 +38,31 @@ while () { my $f = $1; my $ln = $2; - # Makefiles contain nasty expressions to parse docs - next if ($f =~ m/Makefile/); + # Makefiles and scripts contain nasty expressions to parse docs + next if ($f =~ m/Makefile/ || $f =~ m/\.sh$/); + # Skip this script next if ($f eq $scriptname); - if ($ln =~ m,\b(\S*)(Documentation/[A-Za-z0-9\_\.\,\~/\*\[\]\?+-]*),) { + if ($ln =~ m,\b(\S*)(Documentation/[A-Za-z0-9\_\.\,\~/\*\[\]\?+-]*)(.*),) { my $prefix = $1; my $ref = $2; my $base = $2; + my $extra = $3; + + # some file references are like: + # /usr/src/linux/Documentation/DMA-{API,mapping}.txt + # For now, ignore them + next if ($extra =~ m/^{/); + + # Remove footnotes at the end like: + # Documentation/devicetree/dt-object-internal.txt[1] + $ref =~ s/(txt|rst)\[\d+]$/$1/; + + # Remove ending ']' without any '[' + $ref =~ s/\].*// if (!($ref =~ m/\[/)); + # Remove puntuation marks at the end $ref =~ s/[\,\.]+$//; my $fulref = "$prefix$ref"; -- cgit v1.2.3 From a78513c670ac5b5aa1244f93b1833c3e7d5433df Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 14 Jun 2018 11:06:08 -0300 Subject: scripts/documentation-file-ref-check: check tools/*/Documentation Some files, like tools/memory-model/README has references to a Documentation file that is locale to it. Handle references that are relative to them too. Signed-off-by: Mauro Carvalho Chehab Acked-by: Jonathan Corbet --- scripts/documentation-file-ref-check | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'scripts/documentation-file-ref-check') diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index 047f463cdf4be..078999a3fdff7 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -78,6 +78,13 @@ while () { # Check if exists, evaluating wildcards next if (grep -e, glob("$ref $fulref")); + # Accept relative Documentation patches for tools/ + if ($f =~ m/tools/) { + my $path = $f; + $path =~ s,(.*)/.*,$1,; + next if (grep -e, glob("$path/$ref $path/$fulref")); + } + if ($fix) { if (!($ref =~ m/(scripts|Kconfig|Kbuild)/)) { $broken_ref{$ref}++; -- cgit v1.2.3