feat/multi-repo
1package main
2
3import (
4 "bytes"
5 "context"
6 "fmt"
7 "html/template"
8 "os"
9 "path/filepath"
10 "runtime"
11 "strings"
12 "sync"
13
14 "github.com/alecthomas/chroma/v2/formatters/html"
15 "github.com/alecthomas/chroma/v2/lexers"
16 "github.com/alecthomas/chroma/v2/styles"
17
18 "github.com/antonmedv/gitmal/pkg/git"
19 "github.com/antonmedv/gitmal/pkg/links"
20 "github.com/antonmedv/gitmal/pkg/progress_bar"
21 "github.com/antonmedv/gitmal/pkg/templates"
22)
23
24func generateBlobs(files []git.Blob, params Params) error {
25 // Prepare shared, read-only resources
26 var css strings.Builder
27 style := styles.Get(params.Style)
28 if style == nil {
29 return fmt.Errorf("unknown style: %s", params.Style)
30 }
31
32 formatterOptions := []html.Option{
33 html.WithLineNumbers(true),
34 html.WithLinkableLineNumbers(true, "L"),
35 html.WithClasses(true),
36 html.WithCSSComments(false),
37 }
38
39 // Use a temporary formatter to render CSS once
40 var rawCSS strings.Builder
41 if err := html.New(formatterOptions...).WriteCSS(&rawCSS, style); err != nil {
42 return err
43 }
44 css.WriteString(stripChromaBase(rawCSS.String()))
45
46 dirsSet := links.BuildDirSet(files)
47 filesSet := links.BuildFileSet(files)
48
49 // Bounded worker pool
50 workers := runtime.NumCPU()
51 if workers < 1 {
52 workers = 1
53 }
54
55 ctx, cancel := context.WithCancel(context.Background())
56 defer cancel()
57
58 jobs := make(chan git.Blob)
59 errCh := make(chan error, 1)
60 var wg sync.WaitGroup
61
62 p := progress_bar.NewProgressBar("blobs for "+params.Ref.String(), len(files))
63
64 workerFn := func() {
65 defer wg.Done()
66
67 // Per-worker instances
68 md := createMarkdown(params.Style)
69 formatter := html.New(formatterOptions...)
70
71 check := func(err error) bool {
72 if err != nil {
73 select {
74 case errCh <- err:
75 cancel()
76 default:
77 }
78 return true
79 }
80 return false
81 }
82
83 for {
84 select {
85 case <-ctx.Done():
86 return
87 case blob, ok := <-jobs:
88 if !ok {
89 return
90 }
91 func() {
92 var content string
93 data, isBin, err := git.BlobContent(params.Ref, blob.Path, params.RepoDir)
94 if check(err) {
95 return
96 }
97
98 isImg := isImage(blob.Path)
99 if !isBin {
100 content = string(data)
101 }
102
103 outPath := filepath.Join(params.OutputDir, "blob", params.Ref.DirName(), blob.Path) + ".html"
104 if err := os.MkdirAll(filepath.Dir(outPath), 0o755); check(err) {
105 return
106 }
107
108 f, err := os.Create(outPath)
109 if check(err) {
110 return
111 }
112 defer func() {
113 _ = f.Close()
114 }()
115
116 depth := 0
117 if strings.Contains(blob.Path, "/") {
118 depth = len(strings.Split(blob.Path, "/")) - 1
119 }
120 rootHref := strings.Repeat("../", depth+2)
121
122 if isMarkdown(blob.Path) {
123 var b bytes.Buffer
124 if err := md.Convert([]byte(content), &b); check(err) {
125 return
126 }
127
128 contentHTML := links.Resolve(
129 b.String(),
130 blob.Path,
131 rootHref,
132 params.Ref.DirName(),
133 dirsSet,
134 filesSet,
135 )
136
137 err = templates.MarkdownTemplate.ExecuteTemplate(f, "layout.gohtml", templates.MarkdownParams{
138 LayoutParams: templates.LayoutParams{
139 Title: fmt.Sprintf("%s/%s at %s", params.Name, blob.Path, params.Ref),
140 Dark: params.Dark,
141 CSSMarkdown: cssMarkdown(params.Dark),
142 Name: params.Name,
143 SiteName: params.SiteName,
144 RootHref: rootHref + params.RootPrefix,
145 RepoHref: rootHref,
146 CurrentRefDir: params.Ref.DirName(),
147 Selected: "code",
148 InlineStyles: params.InlineStyles,
149 },
150 HeaderParams: templates.HeaderParams{
151 Ref: params.Ref,
152 Breadcrumbs: breadcrumbs(params.Name, blob.Path, true),
153 },
154 Blob: blob,
155 Content: template.HTML(contentHTML),
156 })
157 if check(err) {
158 return
159 }
160
161 } else {
162
163 var contentHTML template.HTML
164 if !isBin {
165 var b bytes.Buffer
166 lx := lexers.Match(blob.Path)
167 if lx == nil {
168 lx = lexers.Fallback
169 }
170 iterator, _ := lx.Tokenise(nil, content)
171 if err := formatter.Format(&b, style, iterator); check(err) {
172 return
173 }
174 contentHTML = template.HTML(b.String())
175
176 } else if isImg {
177
178 rawPath := filepath.Join(params.OutputDir, "raw", params.Ref.DirName(), blob.Path)
179 if err := os.MkdirAll(filepath.Dir(rawPath), 0o755); check(err) {
180 return
181 }
182
183 rf, err := os.Create(rawPath)
184 if check(err) {
185 return
186 }
187 defer func() {
188 _ = rf.Close()
189 }()
190
191 if _, err := rf.Write(data); check(err) {
192 return
193 }
194
195 relativeRawPath := filepath.Join(rootHref, "raw", params.Ref.DirName(), blob.Path)
196 contentHTML = template.HTML(fmt.Sprintf(`<img src="%s" alt="%s" />`, relativeRawPath, blob.FileName))
197 }
198
199 err = templates.BlobTemplate.ExecuteTemplate(f, "layout.gohtml", templates.BlobParams{
200 LayoutParams: templates.LayoutParams{
201 Title: fmt.Sprintf("%s/%s at %s", params.Name, blob.Path, params.Ref),
202 Dark: params.Dark,
203 Name: params.Name,
204 SiteName: params.SiteName,
205 RootHref: rootHref + params.RootPrefix,
206 RepoHref: rootHref,
207 CurrentRefDir: params.Ref.DirName(),
208 Selected: "code",
209 InlineStyles: params.InlineStyles,
210 },
211 HeaderParams: templates.HeaderParams{
212 Ref: params.Ref,
213 Breadcrumbs: breadcrumbs(params.Name, blob.Path, true),
214 },
215 CSS: template.CSS(css.String()),
216 Blob: blob,
217 IsBinary: isBin,
218 IsImage: isImg,
219 Content: contentHTML,
220 })
221 if check(err) {
222 return
223 }
224 }
225 }()
226
227 p.Inc()
228 }
229 }
230 }
231
232 // Start workers
233 wg.Add(workers)
234 for i := 0; i < workers; i++ {
235 go workerFn()
236 }
237
238 // Feed jobs
239 go func() {
240 defer close(jobs)
241 for _, b := range files {
242 select {
243 case <-ctx.Done():
244 return
245 case jobs <- b:
246 }
247 }
248 }()
249
250 // Wait for workers
251 doneCh := make(chan struct{})
252 go func() {
253 wg.Wait()
254 close(doneCh)
255 }()
256
257 var runErr error
258 select {
259 case runErr = <-errCh:
260 // error occurred, wait workers to finish
261 <-doneCh
262 case <-doneCh:
263 }
264
265 p.Done()
266 return runErr
267}