From 3ec595e69fbabc8f61c5b476db02d38c6c5aec2f Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 28 Dec 2020 12:03:36 +0000 Subject: [PATCH 1/3] Profiling fgprof vs pprof. (1609077930 1609077778) Signed-off-by: Bartlomiej Plotka --- go.mod | 5 ++++- go.sum | 30 ++---------------------------- main.go | 22 +++++++++++++++++++--- pkg/mdformatter/mdformatter.go | 3 ++- 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 59b9090..165bb57 100644 --- a/go.mod +++ b/go.mod @@ -19,4 +19,7 @@ require ( gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) -replace github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201225192631-f2e7830d9793 +replace ( + github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201225192631-f2e7830d9793 + github.com/efficientgo/tools => ../tools +) diff --git a/go.sum b/go.sum index 7e801ec..bb7e8d7 100644 --- a/go.sum +++ b/go.sum @@ -80,7 +80,6 @@ github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= @@ -93,9 +92,7 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -120,7 +117,6 @@ github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CL github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -134,20 +130,18 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/efficientgo/tools v0.0.0-20201227131601-4dde9fba2562 h1:fWqPHUxX1zIL0mB4bsydKzQ0+XYpyvKnzzQJ2oRRjVw= -github.com/efficientgo/tools v0.0.0-20201227131601-4dde9fba2562/go.mod h1:jQUsxCcf91LHRhOnGqrx/yrleJbosynzf29/UlCbzlk= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanw/esbuild v0.6.5/go.mod h1:mptxmSXIzBIKKCe4jo9A5SToEd1G+AKZ9JmY85dYRJ0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/fgprof v0.9.1 h1:E6FUJ2Mlv043ipLOCFqo8+cHo9MhQ203E2cdEK/isEs= github.com/felixge/fgprof v0.9.1/go.mod h1:7/HK6JFtFaARhIljgP2IV8rJLIoHDoOYoUphsnGvqxE= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= -github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getkin/kin-openapi v0.14.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= @@ -163,7 +157,6 @@ github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= @@ -205,12 +198,12 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.2-0.20191028172631-481baca67f93/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040 h1:i7RUpu0EybzQyQvPT7J3MmODs4+gPcHsD/pqW0uIYVo= github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -280,12 +273,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/kyokomi/emoji v2.2.1+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA= @@ -305,13 +295,11 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw= @@ -323,7 +311,6 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -378,7 +365,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -453,9 +439,7 @@ github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRci github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tdewolff/minify/v2 v2.6.2/go.mod h1:BkDSm8aMMT0ALGmpt7j3Ra7nLUgZL0qhyrAHXwxcy5w= @@ -498,7 +482,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -524,11 +507,9 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -553,11 +534,9 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -626,14 +605,11 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca h1:pvScuB+UnCGDas2naNKUOXruM08MjwVcEdaweeynIqQ= golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= @@ -679,7 +655,6 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -697,7 +672,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index 6bf96d5..7241119 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" "syscall" + "time" "github.com/bwplotka/mdox/pkg/clilog" "github.com/bwplotka/mdox/pkg/extkingpin" @@ -18,6 +19,8 @@ import ( "github.com/bwplotka/mdox/pkg/mdformatter/linktransformer" "github.com/bwplotka/mdox/pkg/mdformatter/mdgen" "github.com/bwplotka/mdox/pkg/version" + "github.com/efficientgo/tools/pkg/errcapture" + "github.com/efficientgo/tools/pkg/profiles" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -27,7 +30,7 @@ import ( const ( logFormatLogfmt = "logfmt" - logFormatJson = "json" + logFormatJSON = "json" logFormatCLILog = "clilog" ) @@ -46,7 +49,7 @@ func setupLogger(logLevel, logFormat string) log.Logger { panic("unexpected log level") } switch logFormat { - case logFormatJson: + case logFormatJSON: return level.NewFilter(log.NewJSONLogger(log.NewSyncWriter(os.Stderr)), lvl) case logFormatLogfmt: return level.NewFilter(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)), lvl) @@ -62,7 +65,7 @@ func main() { logLevel := app.Flag("log.level", "Log filtering level."). Default("info").Enum("error", "warn", "info", "debug") logFormat := app.Flag("log.format", "Log format to use."). - Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJson, logFormatCLILog) + Default(logFormatCLILog).Enum(logFormatLogfmt, logFormatJSON, logFormatCLILog) ctx, cancel := context.WithCancel(context.Background()) registerFmt(ctx, app) @@ -71,6 +74,17 @@ func main() { cmd, runner := app.Parse() logger := setupLogger(*logLevel, *logFormat) + profileDir := filepath.Join("/home/bwplotka/Repos/_dev/efficientgo/profiles", fmt.Sprintf("%v", time.Now().Unix())) + //close1, err := profiles.StartCPU(profileDir, profiles.CPUTypeBuiltIn) + //if err != nil { + // panic(err) + //} + + closeFn, err := profiles.StartCPU(profileDir, profiles.CPUTypeBuiltIn) + if err != nil { + panic(err) + } + var g run.Group g.Add(func() error { // TODO(bwplotka): Move to customized better setup function. @@ -90,6 +104,7 @@ func main() { } if err := g.Run(); err != nil { + errcapture.CloseWithLog(logger, closeFn, "close1") if *logLevel == "debug" { // Use %+v for github.com/pkg/errors error to print with stack. level.Error(logger).Log("err", fmt.Sprintf("%+v", errors.Wrapf(err, "%s command failed", cmd))) @@ -98,6 +113,7 @@ func main() { level.Error(logger).Log("err", errors.Wrapf(err, "%s command failed", cmd)) os.Exit(1) } + errcapture.CloseWithLog(logger, closeFn, "close1") } func interrupt(logger log.Logger, cancel <-chan struct{}) error { diff --git a/pkg/mdformatter/mdformatter.go b/pkg/mdformatter/mdformatter.go index bcac6b0..178bd60 100644 --- a/pkg/mdformatter/mdformatter.go +++ b/pkg/mdformatter/mdformatter.go @@ -165,7 +165,7 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs if err != nil { return errors.Wrapf(err, "open %v", fn) } - defer errcapture.CloseWithLog(logger, file, "close file %v", fn) + defer errcapture.CloseWithLog(logger, file.Close, "close file %v", fn) b.Reset() if err := f.Format(file, &b); err != nil { @@ -239,6 +239,7 @@ func (f *Formatter) Format(file *os.File, out io.Writer) error { sourceCtx: sourceCtx, link: f.link, cb: f.cb, } + if err := goldmark.New( goldmark.WithExtensions(extension.GFM), goldmark.WithParserOptions(parser.WithAttribute() /* Enable # headers {#custom-ids} */, parser.WithHeadingAttribute()), From 6267082b462180b8bc6645756e510e7b09462df3 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 28 Dec 2020 21:25:05 +0000 Subject: [PATCH 2/3] Updated for new tool. Signed-off-by: Bartlomiej Plotka --- README.md | 3 +- go.mod | 4 +- go.sum | 5 + main.go | 18 +- main_test.go | 2 +- pkg/clilog/clilog.go | 318 ------------------- pkg/mdformatter/linktransformer/link.go | 2 +- pkg/mdformatter/linktransformer/link_test.go | 2 +- pkg/mdformatter/mdformatter.go | 6 +- pkg/mdformatter/mdformatter_test.go | 2 +- pkg/mdformatter/mdgen/mdgen_test.go | 2 +- pkg/mdformatter/transformer.go | 2 +- 12 files changed, 19 insertions(+), 347 deletions(-) delete mode 100644 pkg/clilog/clilog.go diff --git a/README.md b/README.md index b2bfaff..31eb720 100644 --- a/README.md +++ b/README.md @@ -83,12 +83,13 @@ Requirements to build this tool: * Linux or MacOS ```shell -go get github.com/bwplotka/mdox && go mod tidy +go install github.com/bwplotka/mdox ``` or via [bingo](https://github.com/bwplotka/bingo) if want to pin it: ```shell +go install github.com/bwplotka/bingo bingo get -u github.com/bwplotka/mdox ``` diff --git a/go.mod b/go.mod index 165bb57..8d54136 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/Kunde21/markdownfmt/v2 v2.0.4-0.20201214081534-353201c4cdce github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect - github.com/efficientgo/tools v0.0.0-20201227131601-4dde9fba2562 + github.com/efficientgo/tools/core v0.0.0-20201227131601-4dde9fba2562 github.com/go-kit/kit v0.10.0 github.com/gocolly/colly/v2 v2.1.0 github.com/gohugoio/hugo v0.74.3 @@ -21,5 +21,5 @@ require ( replace ( github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201225192631-f2e7830d9793 - github.com/efficientgo/tools => ../tools + github.com/efficientgo/tools/core => ../tools/core ) diff --git a/go.sum b/go.sum index bb7e8d7..bfaa9e7 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,7 @@ github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CL github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -130,6 +131,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/efficientgo/tools v0.0.0-20201228165755-e2b84817bf79 h1:Pi2rMrMQeQ45UAZr8Ple+eSk9WCyglYpkFh0r22E00c= +github.com/efficientgo/tools v0.0.0-20201228165755-e2b84817bf79/go.mod h1:jQUsxCcf91LHRhOnGqrx/yrleJbosynzf29/UlCbzlk= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -365,6 +368,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -482,6 +486,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= diff --git a/main.go b/main.go index 7241119..18a35ad 100644 --- a/main.go +++ b/main.go @@ -11,16 +11,13 @@ import ( "path/filepath" "strings" "syscall" - "time" - "github.com/bwplotka/mdox/pkg/clilog" "github.com/bwplotka/mdox/pkg/extkingpin" "github.com/bwplotka/mdox/pkg/mdformatter" "github.com/bwplotka/mdox/pkg/mdformatter/linktransformer" "github.com/bwplotka/mdox/pkg/mdformatter/mdgen" "github.com/bwplotka/mdox/pkg/version" - "github.com/efficientgo/tools/pkg/errcapture" - "github.com/efficientgo/tools/pkg/profiles" + "github.com/efficientgo/tools/core/pkg/clilog" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -74,17 +71,6 @@ func main() { cmd, runner := app.Parse() logger := setupLogger(*logLevel, *logFormat) - profileDir := filepath.Join("/home/bwplotka/Repos/_dev/efficientgo/profiles", fmt.Sprintf("%v", time.Now().Unix())) - //close1, err := profiles.StartCPU(profileDir, profiles.CPUTypeBuiltIn) - //if err != nil { - // panic(err) - //} - - closeFn, err := profiles.StartCPU(profileDir, profiles.CPUTypeBuiltIn) - if err != nil { - panic(err) - } - var g run.Group g.Add(func() error { // TODO(bwplotka): Move to customized better setup function. @@ -104,7 +90,6 @@ func main() { } if err := g.Run(); err != nil { - errcapture.CloseWithLog(logger, closeFn, "close1") if *logLevel == "debug" { // Use %+v for github.com/pkg/errors error to print with stack. level.Error(logger).Log("err", fmt.Sprintf("%+v", errors.Wrapf(err, "%s command failed", cmd))) @@ -113,7 +98,6 @@ func main() { level.Error(logger).Log("err", errors.Wrapf(err, "%s command failed", cmd)) os.Exit(1) } - errcapture.CloseWithLog(logger, closeFn, "close1") } func interrupt(logger log.Logger, cancel <-chan struct{}) error { diff --git a/main_test.go b/main_test.go index 4ff5e30..97b4ba2 100644 --- a/main_test.go +++ b/main_test.go @@ -7,7 +7,7 @@ import ( "os" "testing" - "github.com/efficientgo/tools/pkg/testutil" + "github.com/efficientgo/tools/core/pkg/testutil" ) func TestValidateAnchorDir(t *testing.T) { diff --git a/pkg/clilog/clilog.go b/pkg/clilog/clilog.go deleted file mode 100644 index 179b4d1..0000000 --- a/pkg/clilog/clilog.go +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright (c) Bartłomiej Płotka @bwplotka -// Licensed under the Apache License 2.0. - -// Package clilog is a github.com/go-kit/kit/log logger implementation suitable for CLI tools. -// Main goal is to log in human readable ways e.g: -// -// * No special sign escaping. -// * No key printing. -// * Values separated with ': ' -// * Support for pretty printing multi errors (including nested ones) in format of (: ; ; ...; ) -// * TODO(bwplotka): Support for multiple multilines. -// This implementation allows CLI to use github.com/go-kit/kit/log enabled libraries in CLI friendly way. -package clilog - -import ( - "bytes" - "encoding" - "errors" - "fmt" - "io" - "reflect" - "sync" - - "github.com/efficientgo/tools/pkg/merrors" - "github.com/go-kit/kit/log" -) - -type buf struct { - bytes.Buffer - - *Encoder -} - -func (l *buf) Reset() { - l.Encoder.Reset() - l.Buffer.Reset() -} - -var bufPool = sync.Pool{ - New: func() interface{} { - var b buf - b.Encoder = NewEncoder(&b.Buffer) - return &b - }, -} - -type logger struct { - w io.Writer -} - -// New returns a logger that encodes keyvals to the Writer in -// CLI friendly format. Each log event produces no more than one call to w.Write. -// The passed Writer must be safe for concurrent use by multiple goroutines if -// the returned Logger will be used concurrently. -func New(w io.Writer) log.Logger { - return &logger{w} -} - -func (l logger) Log(keyvals ...interface{}) error { - buf := bufPool.Get().(*buf) - buf.Reset() - defer bufPool.Put(buf) - - if err := buf.EncodeKeyvals(keyvals...); err != nil { - return err - } - - // Add newline to the end of the buffer - if err := buf.EndRecord(); err != nil { - return err - } - - // The Logger interface requires implementations to be safe for concurrent - // use by multiple goroutines. For this implementation that means making - // only one call to l.w.Write() for each call to Log. - if _, err := l.w.Write(buf.Bytes()); err != nil { - return err - } - return nil -} - -// MarshalKeyvals returns the clilog encoding of keyvals, a variadic sequence -// of alternating keys and values. -func MarshalKeyvals(keyvals ...interface{}) ([]byte, error) { - buf := &bytes.Buffer{} - if err := NewEncoder(buf).EncodeKeyvals(keyvals...); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// An Encoder writes clilog data to an output stream. -type Encoder struct { - w io.Writer - scratch bytes.Buffer - needSep bool - - errs []merrors.Error -} - -// NewEncoder returns a new clilog Encoder that writes to w. -func NewEncoder(w io.Writer) *Encoder { - return &Encoder{w: w} -} - -var ( - sep = []byte(": ") - newline = []byte("\n") - null = []byte("null") -) - -// EncodeKeyval writes the clilog encoding of key and value to the stream. A -// single space is written before the second and subsequent keys in a record. -// Nothing is written if a non-nil error is returned. -func (enc *Encoder) EncodeKeyval(_, value interface{}) error { - if e, ok := value.(error); ok { - if errs, ok := merrors.AsMulti(e); ok { - enc.errs = append(enc.errs, errs) - return nil - } - } - - enc.scratch.Reset() - - // Naive right now, print values only (: - if enc.needSep { - if _, err := enc.scratch.Write(sep); err != nil { - return err - } - } - if err := writeValue(&enc.scratch, value); err != nil { - return err - } - _, err := enc.w.Write(enc.scratch.Bytes()) - enc.needSep = true - return err -} - -// EncodeKeyvals writes the logfmt encoding of keyvals to the stream. Keyvals -// is a variadic sequence of alternating keys and values. Keys of unsupported -// type are skipped along with their corresponding value. Values of -// unsupported type or that cause a MarshalerError are replaced by their error -// but do not cause EncodeKeyvals to return an error. If a non-nil error is -// returned some key/value pairs may not have be written. -func (enc *Encoder) EncodeKeyvals(keyvals ...interface{}) error { - if len(keyvals) == 0 { - return nil - } - if len(keyvals)%2 == 1 { - keyvals = append(keyvals, nil) - } - for i := 0; i < len(keyvals); i += 2 { - k, v := keyvals[i], keyvals[i+1] - err := enc.EncodeKeyval(k, v) - if err == ErrUnsupportedKeyType { - continue - } - if _, ok := err.(*MarshalerError); ok || err == ErrUnsupportedValueType { - v = err - err = enc.EncodeKeyval(k, v) - } - if err != nil { - return err - } - } - return nil -} - -// EndRecord ends the log record. -func (enc *Encoder) EndRecord() error { - if len(enc.errs) > 0 { - enc.scratch.Reset() - if enc.needSep { - if _, err := enc.scratch.Write(sep); err != nil { - return err - } - } - - merr := merrors.Merge(enc.errs) - if err := merrors.PrettyPrint(&enc.scratch, merr); err != nil { - return err - } - - if _, err := enc.w.Write(enc.scratch.Bytes()); err != nil { - return err - } - } - - _, err := enc.w.Write(newline) - if err == nil { - enc.needSep = false - } - return err -} - -// Reset resets the Encoder to the beginning of a new record. -func (enc *Encoder) Reset() { - enc.needSep = false -} - -// MarshalerError represents an error encountered while marshaling a value. -type MarshalerError struct { - Type reflect.Type - Err error -} - -func (e *MarshalerError) Error() string { - return "error marshaling value of type " + e.Type.String() + ": " + e.Err.Error() -} - -// ErrUnsupportedKeyType is returned by Encoder methods if a key has an -// unsupported type. -var ErrUnsupportedKeyType = errors.New("unsupported key type") - -// ErrUnsupportedValueType is returned by Encoder methods if a value has an -// unsupported type. -var ErrUnsupportedValueType = errors.New("unsupported value type") - -func writeValue(w io.Writer, value interface{}) error { - switch v := value.(type) { - case nil: - return writeBytesValue(w, null) - case string: - return writeStringValue(w, v, true) - case []byte: - return writeBytesValue(w, v) - case encoding.TextMarshaler: - vb, err := safeMarshal(v) - if err != nil { - return err - } - if vb == nil { - vb = null - } - return writeBytesValue(w, vb) - case error: - se, ok := safeError(v) - return writeStringValue(w, se, ok) - case fmt.Stringer: - ss, ok := safeString(v) - return writeStringValue(w, ss, ok) - default: - rvalue := reflect.ValueOf(value) - switch rvalue.Kind() { - case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.Slice, reflect.Struct: - return ErrUnsupportedValueType - case reflect.Ptr: - if rvalue.IsNil() { - return writeBytesValue(w, null) - } - return writeValue(w, rvalue.Elem().Interface()) - } - return writeStringValue(w, fmt.Sprint(v), true) - } -} - -func writeStringValue(w io.Writer, value string, ok bool) error { - var err error - if ok && value == "null" { - _, err = io.WriteString(w, `"null"`) - } else { - _, err = io.WriteString(w, value) - } - return err -} - -func writeBytesValue(w io.Writer, value []byte) error { - _, err := w.Write(value) - return err -} - -func safeError(err error) (s string, ok bool) { - defer func() { - if panicVal := recover(); panicVal != nil { - if v := reflect.ValueOf(err); v.Kind() == reflect.Ptr && v.IsNil() { - s, ok = "null", false - } else { - s, ok = fmt.Sprintf("PANIC:%v", panicVal), false - } - } - }() - s, ok = err.Error(), true - return -} - -func safeString(str fmt.Stringer) (s string, ok bool) { - defer func() { - if panicVal := recover(); panicVal != nil { - if v := reflect.ValueOf(str); v.Kind() == reflect.Ptr && v.IsNil() { - s, ok = "null", false - } else { - s, ok = fmt.Sprintf("PANIC:%v", panicVal), true - } - } - }() - s, ok = str.String(), true - return -} - -func safeMarshal(tm encoding.TextMarshaler) (b []byte, err error) { - defer func() { - if panicVal := recover(); panicVal != nil { - if v := reflect.ValueOf(tm); v.Kind() == reflect.Ptr && v.IsNil() { - b, err = nil, nil - } else { - b, err = nil, fmt.Errorf("panic when marshaling: %s", panicVal) - } - } - }() - b, err = tm.MarshalText() - if err != nil { - return nil, &MarshalerError{ - Type: reflect.TypeOf(tm), - Err: err, - } - } - return -} diff --git a/pkg/mdformatter/linktransformer/link.go b/pkg/mdformatter/linktransformer/link.go index 33f7157..e193c30 100644 --- a/pkg/mdformatter/linktransformer/link.go +++ b/pkg/mdformatter/linktransformer/link.go @@ -15,7 +15,7 @@ import ( "sync" "github.com/bwplotka/mdox/pkg/mdformatter" - "github.com/efficientgo/tools/pkg/merrors" + "github.com/efficientgo/tools/core/pkg/merrors" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/gocolly/colly/v2" diff --git a/pkg/mdformatter/linktransformer/link_test.go b/pkg/mdformatter/linktransformer/link_test.go index 7539849..55b9628 100644 --- a/pkg/mdformatter/linktransformer/link_test.go +++ b/pkg/mdformatter/linktransformer/link_test.go @@ -13,7 +13,7 @@ import ( "testing" "github.com/bwplotka/mdox/pkg/mdformatter" - "github.com/efficientgo/tools/pkg/testutil" + "github.com/efficientgo/tools/core/pkg/testutil" "github.com/go-kit/kit/log" ) diff --git a/pkg/mdformatter/mdformatter.go b/pkg/mdformatter/mdformatter.go index 178bd60..52aa25e 100644 --- a/pkg/mdformatter/mdformatter.go +++ b/pkg/mdformatter/mdformatter.go @@ -14,8 +14,8 @@ import ( "github.com/Kunde21/markdownfmt/v2/markdown" "github.com/bwplotka/mdox/pkg/gitdiff" - "github.com/efficientgo/tools/pkg/errcapture" - "github.com/efficientgo/tools/pkg/merrors" + "github.com/efficientgo/tools/core/pkg/logerrcapture" + "github.com/efficientgo/tools/core/pkg/merrors" "github.com/go-kit/kit/log" "github.com/gohugoio/hugo/parser/pageparser" "github.com/pkg/errors" @@ -165,7 +165,7 @@ func format(ctx context.Context, logger log.Logger, files []string, diffs *Diffs if err != nil { return errors.Wrapf(err, "open %v", fn) } - defer errcapture.CloseWithLog(logger, file.Close, "close file %v", fn) + defer logerrcapture.Close(logger, file.Close, "close file %v", fn) b.Reset() if err := f.Format(file, &b); err != nil { diff --git a/pkg/mdformatter/mdformatter_test.go b/pkg/mdformatter/mdformatter_test.go index f7cd061..8cccfbe 100644 --- a/pkg/mdformatter/mdformatter_test.go +++ b/pkg/mdformatter/mdformatter_test.go @@ -10,7 +10,7 @@ import ( "os" "testing" - "github.com/efficientgo/tools/pkg/testutil" + "github.com/efficientgo/tools/core/pkg/testutil" "github.com/go-kit/kit/log" ) diff --git a/pkg/mdformatter/mdgen/mdgen_test.go b/pkg/mdformatter/mdgen/mdgen_test.go index 12f3fa3..170bcc8 100644 --- a/pkg/mdformatter/mdgen/mdgen_test.go +++ b/pkg/mdformatter/mdgen/mdgen_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/bwplotka/mdox/pkg/mdformatter" - "github.com/efficientgo/tools/pkg/testutil" + "github.com/efficientgo/tools/core/pkg/testutil" ) func TestFormat_FormatSingle_CodeBlockTransformer(t *testing.T) { diff --git a/pkg/mdformatter/transformer.go b/pkg/mdformatter/transformer.go index 0831578..4f15e82 100644 --- a/pkg/mdformatter/transformer.go +++ b/pkg/mdformatter/transformer.go @@ -7,7 +7,7 @@ import ( "bytes" "io" - "github.com/efficientgo/tools/pkg/merrors" + "github.com/efficientgo/tools/core/pkg/merrors" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/renderer" "github.com/yuin/goldmark/text" From 887073665a15904e1799971cd0f754384b2d09a3 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 30 Dec 2020 15:23:21 +0000 Subject: [PATCH 3/3] Attempt of instrumenting. Signed-off-by: Bartlomiej Plotka --- go.mod | 4 +- go.sum | 8 ++++ main.go | 54 +++++++++++++++++++++++++ pkg/mdformatter/linktransformer/link.go | 20 ++++++++- 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8d54136..b2d3ec8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/Kunde21/markdownfmt/v2 v2.0.4-0.20201214081534-353201c4cdce github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect - github.com/efficientgo/tools/core v0.0.0-20201227131601-4dde9fba2562 + github.com/efficientgo/tools/core v0.0.0-20201228212819-69909db83cda github.com/go-kit/kit v0.10.0 github.com/gocolly/colly/v2 v2.1.0 github.com/gohugoio/hugo v0.74.3 @@ -13,6 +13,8 @@ require ( github.com/mitchellh/mapstructure v1.2.2 // indirect github.com/oklog/run v1.1.0 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.3.0 + github.com/prometheus/common v0.7.0 github.com/sergi/go-diff v1.0.0 github.com/yuin/goldmark v1.2.1 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect diff --git a/go.sum b/go.sum index bfaa9e7..6f35627 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,7 @@ github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY= @@ -92,7 +93,9 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -303,6 +306,7 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/mmark v1.3.6/go.mod h1:w7r9mkTvpS55jlfyn22qJ618itLryxXBhA7Jp3FIlkw= @@ -375,21 +379,25 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= diff --git a/main.go b/main.go index 18a35ad..55859ce 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,8 @@ package main import ( "context" "fmt" + "net/http" + "net/url" "os" "os/signal" "path/filepath" @@ -18,10 +20,14 @@ import ( "github.com/bwplotka/mdox/pkg/mdformatter/mdgen" "github.com/bwplotka/mdox/pkg/version" "github.com/efficientgo/tools/core/pkg/clilog" + "github.com/efficientgo/tools/core/pkg/errcapture" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/expfmt" "gopkg.in/alecthomas/kingpin.v2" ) @@ -150,11 +156,39 @@ This directive runs executable with arguments and put its stderr and stdout outp } var linkTr []mdformatter.LinkTransformer + var m *httpMetrics if *linksValidateEnabled { v, err := linktransformer.NewValidator(logger, *linksValidateExceptDomains, anchorDir) if err != nil { return err } + + m = &httpMetrics{ + reg: prometheus.NewRegistry(), + } + requests := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "does_not_matter"}, + []string{"domain", "code", "method"}, + ) + perDomainLatency := prometheus.NewHistogramVec( + prometheus.HistogramOpts{Name: "does_not_matter1", Buckets: prometheus.DefBuckets}, + []string{"domain", "code", "method"}, + ) + m.reg.MustRegister(requests, perDomainLatency) + defer errcapture.Close(&err, m.Print, "print") + + v.SetTransportFunc(func(u string) http.RoundTripper { + parsed, err := url.Parse(u) + if err != nil { + panic(err) + } + return promhttp.InstrumentRoundTripperCounter( + requests, + promhttp.InstrumentRoundTripperDuration(perDomainLatency.MustCurryWith(prometheus.Labels{ + "domain": parsed.Host, + }), http.DefaultTransport), + ) + }) linkTr = append(linkTr, v) } if *linksLocalizeForAddress != nil { @@ -179,6 +213,26 @@ This directive runs executable with arguments and put its stderr and stdout outp }) } +type httpMetrics struct { + reg *prometheus.Registry +} + +func (h *httpMetrics) Print() error { + mfs, err := h.reg.Gather() + if err != nil { + return err + } + + enc := expfmt.NewEncoder(os.Stdout, expfmt.FmtProtoText) + + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + return err + } + } + return nil +} + // validateAnchorDir returns validated anchor dir against files provided. func validateAnchorDir(anchorDir string, files []string) (_ string, err error) { if anchorDir == "" { diff --git a/pkg/mdformatter/linktransformer/link.go b/pkg/mdformatter/linktransformer/link.go index e193c30..e54ef2f 100644 --- a/pkg/mdformatter/linktransformer/link.go +++ b/pkg/mdformatter/linktransformer/link.go @@ -7,6 +7,7 @@ import ( "bufio" "bytes" "io" + "net/http" "os" "path/filepath" "regexp" @@ -20,6 +21,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/gocolly/colly/v2" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" ) var remoteLinkPrefixRe = regexp.MustCompile(`^http[s]?://`) @@ -117,6 +119,8 @@ type validator struct { futureMu sync.Mutex destFutures map[futureKey]*futureResult + + transportFn func(url string) http.RoundTripper } type futureKey struct { @@ -131,7 +135,7 @@ type futureResult struct { // NewValidator returns mdformatter.LinkTransformer that crawls all links. // TODO(bwplotka): Add optimization and debug modes - this is the main source of latency and pain. -func NewValidator(logger log.Logger, except *regexp.Regexp, anchorDir string) (mdformatter.LinkTransformer, error) { +func NewValidator(logger log.Logger, except *regexp.Regexp, anchorDir string) (*validator, error) { v := &validator{ logger: logger, anchorDir: anchorDir, @@ -140,6 +144,9 @@ func NewValidator(logger log.Logger, except *regexp.Regexp, anchorDir string) (m remoteLinks: map[string]error{}, c: colly.NewCollector(colly.Async()), destFutures: map[futureKey]*futureResult{}, + transportFn: func(url string) http.RoundTripper { + return http.DefaultTransport + }, } // Set very soft limits. // E.g github has 50-5000 https://docs.github.com/en/free-pro-team@latest/rest/reference/rate-limit limit depending @@ -246,11 +253,20 @@ func (v *validator) visit(filepath string, dest string) { return } - if err := v.c.Visit(dest); err != nil { + if err := v.c.WithTransport(v.transportFn(dest)).Visit(dest); err != nil { v.remoteLinks[dest] = errors.Wrapf(err, "remote link %v", dest) } } +type Metrics struct { + requests *prometheus.CounterVec + latency *prometheus.HistogramVec +} + +func (v *validator) SetTransportFunc(transportFn func(url string) http.RoundTripper) { + v.transportFn = transportFn +} + type localLinksCache map[string]*[]string // Lookup looks for given link in local anchorDir. It returns error if link can't be found.