From 0224a259c7fd61fbabdb8ab632471e68b7fd6b4a Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Thu, 6 Jun 2019 10:48:40 -0700 Subject: Add script used to generate devstats repo groups This isn't perfect, but it's probably better living here than in a random gist I have laying around. --- hack/generate-devstats-repo-sql.py | 91 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100755 hack/generate-devstats-repo-sql.py (limited to 'hack') diff --git a/hack/generate-devstats-repo-sql.py b/hack/generate-devstats-repo-sql.py new file mode 100755 index 00000000..8ce88c21 --- /dev/null +++ b/hack/generate-devstats-repo-sql.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Output devstats repo_groups.sql based on subproject defintions in sigs.yaml + +This is likely missing a few repos because: + - some repos lack an owner (eg: kubernetes/kubernetes) + - it doesn't enumerate all repos from all kubernetes-owned orgs + - it ignores the fact that committees can own repos, only grouping by sig + +The sql generated is NOT intended to overwrite/replace the file that lives at +github.com/cncf/devstats/scripts/kubernetes/repo_groups.sql, but instead aid a +human in doing some manual updates to the file. Future improvements to this +script could eliminate that part of the process, but it's where we are today. +""" + +import argparse +import ruamel.yaml as yaml +import json +import re +import sys + +update_gha_repos_template = """ +update gha_repos set repo_group = 'SIG {}' where name in ( +{} +); +""" + +def repos_from_sig(sig): + """Returns a list of org/repos given a sig""" + repos = {} + subprojects = sig.get('subprojects', []) + if subprojects is None: + subprojects = [] + for sp in subprojects: + for uri in sp['owners']: + owners_path = re.sub(r"https://raw.githubusercontent.com/(.*)/master/(.*)",r"\1/\2",uri) + path_parts = owners_path.split('/') + # org/repo is owned by sig if org/repo/OWNERS os in one of their subprojects + if path_parts[2] == 'OWNERS': + repo = '/'.join(path_parts[0:2]) + repos[repo] = True + return sorted(repos.keys()) + +def write_repo_groups_sql(sigs, fp): + for sig in sigs['sigs']: + repos = repos_from_sig(sig) + if len(repos): + fp.write( + update_gha_repos_template.format( + sig['name'], + ',\n'.join([' \'{}\''.format(r) for r in repos]))) + +def main(sigs_yaml, repo_groups_sql): + with open(sigs_yaml) as fp: + sigs = yaml.round_trip_load(fp) + + if repo_groups_sql is not None: + with open(repo_groups_sql, 'w') as fp: + write_repo_groups_sql(sigs, fp) + else: + write_repo_groups_sql(sigs, sys.stdout) + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser( + description='Do things with sigs.yaml') + PARSER.add_argument( + '--sigs-yaml', + default='./sigs.yaml', + help='Path to sigs.yaml') + PARSER.add_argument( + '--repo-groups-sql', + help='Path to output repo_groups.sql if provided') + ARGS = PARSER.parse_args() + + main(ARGS.sigs_yaml, ARGS.repo_groups_sql) + -- cgit v1.2.3 From 7bd7f2c34468c888a1cacd54d35d5a33aea2ee4c Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Wed, 26 Jun 2019 13:05:43 -0700 Subject: Add more details to generate-devstats-repo-sql.py There was still a lot of manual massaging of the script's output that needed to happen before it was usable as a PR to cncf/devstats. I have now tried to encode as much of that knowledge in code and comments here to allow the output of the script to overwrite the existing file. - add header/footer - add comment pointing to this script - add special case group for "Kubernetes" - add support for committees that own code - (manually) keep track of old repo names for history --- hack/generate-devstats-repo-sql.py | 139 ++++++++++++++++++++++++++++++++----- 1 file changed, 121 insertions(+), 18 deletions(-) (limited to 'hack') diff --git a/hack/generate-devstats-repo-sql.py b/hack/generate-devstats-repo-sql.py index 8ce88c21..71d77497 100755 --- a/hack/generate-devstats-repo-sql.py +++ b/hack/generate-devstats-repo-sql.py @@ -34,50 +34,153 @@ import json import re import sys -update_gha_repos_template = """ -update gha_repos set repo_group = 'SIG {}' where name in ( +repo_group_sql_template = """ +update gha_repos set repo_group = '{}' where name in ( {} ); """ -def repos_from_sig(sig): - """Returns a list of org/repos given a sig""" +# copied from github.com/cncf/devstats/scripts/kubernetes/repo_groups.sql, +# if this differs, consider cncf the authoritative source and update this +repo_groups_sql_header = """-- generated by github.com/kubernetes/community/hack/generate-devstats-repo-sql.py +-- Add repository groups +""" + +# copied from github.com/cncf/devstats/scripts/kubernetes/repo_groups.sql, +# if this differs, consider cncf the authoritative source and update this +repo_groups_sql_footer = """ +-- All other unknown repositories should have 'Other' repository group +-- update gha_repos set repo_group = 'Other' where repo_group is null; + +-- By default alias is the newest repo name for given repo ID +update + gha_repos r +set + alias = coalesce(( + select e.dup_repo_name + from + gha_events e + where + e.repo_id = r.id + order by + e.created_at desc + limit 1 + ), name) +; + +update gha_repos set alias = 'kubernetes/kubernetes' where name like '%kubernetes' or name = 'kubernetes/'; + +select + repo_group, + count(*) as number_of_repos +from + gha_repos +where + repo_group is not null +group by + repo_group +order by + number_of_repos desc, + repo_group asc; + +""" + +special_case_groups = [{ + # the main repo has no single owner and has gone by many names + 'name': 'Kubernetes', + 'repos': [ + 'kubernetes/kubernetes', + 'GoogleCloudPlatform/kubernetes', + 'kubernetes', + 'kubernetes/' + ] +}] + +# devstats isn't aware of repo renames or migrations; we need to keep +# old repo names in its sql groups present for historical purposes; +# +# when reconciling deletions from repo_groups.sql by this script, use +# github.com/kubernetes/org issues to determine why; renamed, migrated, +# or used-and-retired repos belong here; unused/deleted repos do not +renamed_repos = { + 'sig-architecture': [ + 'kubernetes/contrib', + ], + 'sig-api-machinery': [ + 'kubernetes-incubator/apiserver-builder', + ], + 'sig-cluster-lifecycle': [ + 'kubernetes-incubator/kubespray', + ], + 'sig-multicluster': [ + 'kubernetes-sigs/federation-v2', + ], + 'sig-node': [ + 'kubernetes-incubator/node-feature-discovery', + ], + 'sig-pm': [ + 'kubernetes/features', + ], + 'sig-service-catalog': [ + 'kubernetes-incubator/service-catalog', + ] +} + +def repos_from_k8s_group(k8s_group): + """Returns a list of org/repos given a kubernetes community group""" repos = {} - subprojects = sig.get('subprojects', []) + subprojects = k8s_group.get('subprojects', []) if subprojects is None: subprojects = [] for sp in subprojects: for uri in sp['owners']: owners_path = re.sub(r"https://raw.githubusercontent.com/(.*)/master/(.*)",r"\1/\2",uri) path_parts = owners_path.split('/') - # org/repo is owned by sig if org/repo/OWNERS os in one of their subprojects + # org/repo is owned by k8s_group if org/repo/OWNERS os in one of their subprojects if path_parts[2] == 'OWNERS': repo = '/'.join(path_parts[0:2]) repos[repo] = True return sorted(repos.keys()) -def write_repo_groups_sql(sigs, fp): - for sig in sigs['sigs']: - repos = repos_from_sig(sig) - if len(repos): - fp.write( - update_gha_repos_template.format( - sig['name'], - ',\n'.join([' \'{}\''.format(r) for r in repos]))) +def k8s_group_name(k8s_group): + group_dir = k8s_group.get('dir', '') + if group_dir.startswith('sig-'): + return "SIG " + k8s_group['name'] + if group_dir.startswith('committee-'): + return k8s_group['name'] + " Committee" + return "UNKNOWN " + group_dir + +def write_repo_groups_template(name, repos, fp): + if len(repos): + fp.write( + repo_group_sql_template.format( + name, + ',\n'.join([' \'{}\''.format(r) for r in repos]))) + +def write_repo_groups_sql(k8s_groups, fp): + fp.write(repo_groups_sql_header) + for g in special_case_groups: + write_repo_groups_template(g['name'], g['repos'], fp) + for group_type in ['sigs', 'committees']: + for g in k8s_groups[group_type]: + repos = set(repos_from_k8s_group(g)) | set(renamed_repos.get(g['dir'],[])) + repos = sorted(list(repos)) + write_repo_groups_template(k8s_group_name(g), repos, fp) + fp.write(repo_groups_sql_footer) def main(sigs_yaml, repo_groups_sql): with open(sigs_yaml) as fp: - sigs = yaml.round_trip_load(fp) + k8s_groups = yaml.round_trip_load(fp) if repo_groups_sql is not None: with open(repo_groups_sql, 'w') as fp: - write_repo_groups_sql(sigs, fp) + write_repo_groups_sql(k8s_groups, fp) else: - write_repo_groups_sql(sigs, sys.stdout) + write_repo_groups_sql(k8s_groups, sys.stdout) if __name__ == '__main__': PARSER = argparse.ArgumentParser( - description='Do things with sigs.yaml') + description='Generate a repo_groups.sql intended for github.com/cncf/devstats/scripts/kubernetes/repo_groups.sql') PARSER.add_argument( '--sigs-yaml', default='./sigs.yaml', -- cgit v1.2.3 From 91632198e4a71da70ab1448ef895cf2e77b59e75 Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Sun, 28 Jul 2019 13:54:39 -0700 Subject: Use versioned misspell from go.mod --- hack/verify-spelling.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'hack') diff --git a/hack/verify-spelling.sh b/hack/verify-spelling.sh index 1831f8ba..f615a323 100755 --- a/hack/verify-spelling.sh +++ b/hack/verify-spelling.sh @@ -19,19 +19,15 @@ set -o nounset set -o pipefail export KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +export GO111MODULE=on +export GOPROXY="${GOPROXY:-https://proxy.golang.org}" -# Install tools we need, but only from vendor/... -cd ${KUBE_ROOT} -go install ./vendor/github.com/client9/misspell/cmd/misspell -if ! which misspell >/dev/null 2>&1; then - echo "Can't find misspell - is your GOPATH 'bin' in your PATH?" >&2 - echo " GOPATH: ${GOPATH}" >&2 - echo " PATH: ${PATH}" >&2 - exit 1 -fi +# Pick out version of misspell from go.mod +go mod download +misspell="$(go list -m -f '{{.Dir}}' github.com/client9/misspell)" # Spell checking # All the skipping files are defined in hack/.spelling_failures skipping_file="${KUBE_ROOT}/hack/.spelling_failures" failing_packages=$(echo `cat ${skipping_file}` | sed "s| | -e |g") -git ls-files | grep -v -e ${failing_packages} | xargs misspell -i "" -error -o stderr +git ls-files | grep -v -e ${failing_packages} | xargs go run "${misspell}/cmd/misspell" -i "" -error -o stderr -- cgit v1.2.3 From 436f329e9dc6dad7338fab6f1609b9683817ea92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Tue, 30 Apr 2019 10:05:40 +0200 Subject: Add a French translation of contributor-cheatsheet --- hack/.spelling_failures | 1 + 1 file changed, 1 insertion(+) (limited to 'hack') diff --git a/hack/.spelling_failures b/hack/.spelling_failures index 0cafda99..528c772d 100644 --- a/hack/.spelling_failures +++ b/hack/.spelling_failures @@ -2,4 +2,5 @@ events/elections/2017/ vendor/ sig-contributor-experience/contribex-survey-2018.csv events/2014 +contributors/guide/contributor-cheatsheet/README-fr.md contributors/guide/contributor-cheatsheet/README-pt.md -- cgit v1.2.3 From 9f3bd17a088d54cff6a0de1c6630d50f9bca50ca Mon Sep 17 00:00:00 2001 From: Charlotte Date: Tue, 27 Aug 2019 18:14:30 +0200 Subject: Add German translation of contributor cheatsheet --- hack/.spelling_failures | 1 + 1 file changed, 1 insertion(+) (limited to 'hack') diff --git a/hack/.spelling_failures b/hack/.spelling_failures index 528c772d..22fdbb0a 100644 --- a/hack/.spelling_failures +++ b/hack/.spelling_failures @@ -4,3 +4,4 @@ sig-contributor-experience/contribex-survey-2018.csv events/2014 contributors/guide/contributor-cheatsheet/README-fr.md contributors/guide/contributor-cheatsheet/README-pt.md +contributors/guide/contributor-cheatsheet/README-de.md -- cgit v1.2.3