1 import com.trendmicro.grid.acl.client.ServiceClient;
2 import com.trendmicro.grid.acl.client.util.CommandlineClientFactory;
3 import com.trendmicro.grid.acl.client.util.CommandlineParser;
4 import com.trendmicro.grid.acl.l0.AuthenticationException;
5 import com.trendmicro.grid.acl.l0.BatchCollection;
6 import com.trendmicro.grid.acl.l0.PackageService;
7 import com.trendmicro.grid.acl.l0.ProcessingService;
8 import com.trendmicro.grid.acl.l0.datatypes.DaysRange;
9 import com.trendmicro.grid.acl.l0.datatypes.FileIdentifier;
10 import com.trendmicro.grid.acl.l0.datatypes.NameListPage;
11
12 import java.text.DateFormat;
13 import java.text.SimpleDateFormat;
14 import java.util.*;
15 import java.util.concurrent.ExecutorService;
16 import java.util.concurrent.Executors;
17 import java.util.concurrent.TimeUnit;
18
19 import static com.trendmicro.grid.acl.l0.datatypes.DaysRange.RangeType;
20 import static java.lang.System.*;
21
22
23
24
25
26
27
28
29 public class ReprocessInitiator {
30
31 static final ExecutorService executorService =
32 Executors.newFixedThreadPool(Integer.getInteger("max.connections", 4));
33
34 static final String[] HELP = {
35 "GRID Reprocess Initiator - Version 1.0",
36 "",
37 "Selects a list of packages match the specified tag query and optional time range",
38 "and forces their reprocessing using the source & processing SOAP API.",
39 "",
40 };
41
42 static final CommandlineParser CLI = new CommandlineParser().
43
44 defineParameter("Defines the tag query to use for identifying the packages.\n" +
45 "Tag queries are of the format:\n" +
46 " \"tag01 tag01 -excludedTag01 -excludedTag02\"\n" +
47 " \"(g1tag -g1extag) (g2tag -g2extag)\"",
48 false, String.class, null, "-q", "--query").
49
50 defineParameter("Defines an optional 'prefix' that must appear in the package name.",
51 false, String.class, null, "--prefix").
52
53 defineParameter("Defines an optional 'from' date using YYYY-MM-DD.",
54 false, String.class, null, "-f", "--from").
55 defineParameter("Defines an optional 'to' date using YYYY-MM-DD.",
56 false, String.class, null, "-t", "--to").
57 defineParameter("Defines how to apply dates, can be one of: " +
58 "'FIRST_SEEN', 'LAST_RETRIEVED' or 'LAST_PROCESSED'",
59 false, String.class, "FIRST_SEEN", "--range-type").
60
61 defineSwitchParameter("List package names only, DO NOT start reprocessing.",
62 "-l", "--list");
63
64 static final CommandlineClientFactory CLIENT_FACTORY = new CommandlineClientFactory(CLI);
65
66
67 public static void main(String[] args) throws Exception {
68 boolean canContinue = CLI.parse(args, out, HELP);
69 if (!canContinue)
70 return;
71
72 final ServiceClient client = CLIENT_FACTORY.newServiceClient(CLI);
73
74 long count = 0;
75 final long time = currentTimeMillis();
76 try {
77 final DaysRange range;
78 String fromDate = CLI.getParameter("-f", String.class), toDate = CLI.getParameter("-t", String.class);
79 if (fromDate != null || toDate != null) {
80 final DateFormat df = new SimpleDateFormat("yyyy-MM-dd");
81 final RangeType type = RangeType.valueOf(CLI.getParameter("--range-type", String.class));
82 range = new DaysRange(type,
83 fromDate == null ? null : df.parse(fromDate),
84 toDate == null ? null : df.parse(toDate));
85 } else
86 range = null;
87
88 count = queryPackages(client,
89 CLI.getParameter("-q", String.class), CLI.getParameter("--prefix", String.class), range);
90 } finally {
91 executorService.shutdown();
92 while (!executorService.awaitTermination(30, TimeUnit.SECONDS))
93 out.println("Waiting on pending transfers.");
94
95 out.println();
96 out.printf("Packages: %d, Time: %.3f sec %n", count, ((double) (currentTimeMillis() - time)) / 1000D);
97 }
98 }
99
100 static long queryPackages(ServiceClient client, String tagQuery, String namePrefix, DaysRange range) throws AuthenticationException {
101 final PackageService packageService = client.getPort(PackageService.class);
102
103 int pageNumber = 0;
104 long totalCount = 0, skippedCount = 0;
105 NameListPage namePage;
106 int maxClusterSize = Integer.getInteger("max.chunksize", 16);
107 namePrefix = namePrefix == null ? null : namePrefix.toLowerCase();
108
109 do {
110 namePage = range == null ?
111 packageService.getMatchingPackageNames(tagQuery, "1.0", pageNumber) :
112 packageService.getMatchingPackageNamesInRange(tagQuery, "1.0", range, pageNumber);
113 if (namePage == null)
114 break;
115 pageNumber++;
116
117 NameHandler fh = null;
118 for (String packageName : namePage.getElements()) {
119 if (namePrefix != null && !packageName.startsWith(namePrefix)) {
120 skippedCount++;
121 continue;
122 }
123 while (fh == null || !fh.addIfSpaceAvailable(packageName, maxClusterSize)) {
124 if (fh != null)
125 executorService.execute(fh);
126 fh = new NameHandler(client);
127 }
128 }
129
130 if (fh != null)
131 executorService.execute(fh);
132
133 totalCount += namePage.getElements().size();
134
135 } while (!namePage.isLastPage());
136
137 return totalCount - skippedCount;
138 }
139
140 private static class NameHandler implements Runnable {
141
142 final boolean listOnly = CLI.isParameterTrue("-l");
143
144 final private List<String> names = new ArrayList<String>();
145
146 final PackageService packageService;
147 final ProcessingService processingService;
148
149 private NameHandler(ServiceClient client) {
150 packageService = client.getPort(PackageService.class);
151 processingService = client.getPort(ProcessingService.class);
152 }
153
154 boolean addIfSpaceAvailable(String name, int maxSize) {
155 return names.size() < maxSize && names.add(name);
156 }
157
158
159
160
161 public void run() {
162
163 final Iterator<String> nameIterator = names.iterator();
164 final Collection<FileIdentifier> fileIds;
165 try {
166 fileIds = packageService.getPackageFileIdentifiersByName(BatchCollection.of(names));
167 } catch (AuthenticationException e) {
168 throw new RuntimeException(e);
169 }
170
171 for (FileIdentifier fileId : fileIds) {
172 final String packageName = nameIterator.next();
173 if (fileId == null) {
174 err.println("Failed to find file content for package '" + packageName +
175 "', cannot re-process it.");
176 continue;
177 }
178
179 try {
180 if (listOnly) {
181 out.println("Would initiate the re-processing of package '" +
182 packageName + "' with file '" + fileId + "'");
183 } else {
184 UUID jobId = processingService.startReprocessingJob(fileId, false, -1);
185 synchronized (out) {
186
187
188
189 out.println("Initiated the re-processing of package '" + packageName + "'");
190 out.println(" (Job-ID: '" + jobId + "' with file '" + fileId + "')");
191 }
192 }
193 } catch (Exception e) {
194 err.println("Failed initiating the re-processing of package '" + packageName + "'");
195 e.printStackTrace();
196 }
197 }
198 }
199 }
200 }