serposcope.controllers.google.GoogleTargetController Maven / Gradle / Ivy
The newest version!
/*
* Serposcope - SEO rank checker https://serposcope.serphacker.com/
*
* Copyright (c) 2016 SERP Hacker
* @author Pierre Nogues
* @license https://opensource.org/licenses/MIT MIT License
*/
package serposcope.controllers.google;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Functions;
import com.google.inject.Inject;
import ninja.Result;
import ninja.Results;
import com.google.inject.Singleton;
import com.serphacker.serposcope.db.base.BaseDB;
import com.serphacker.serposcope.db.base.RunDB;
import com.serphacker.serposcope.db.google.GoogleDB;
import com.serphacker.serposcope.models.base.Config;
import com.serphacker.serposcope.models.base.Event;
import com.serphacker.serposcope.models.base.Group;
import com.serphacker.serposcope.models.base.Run;
import com.serphacker.serposcope.models.google.GoogleBest;
import com.serphacker.serposcope.models.google.GoogleRank;
import static com.serphacker.serposcope.models.google.GoogleRank.UNRANKED;
import com.serphacker.serposcope.models.google.GoogleSearch;
import com.serphacker.serposcope.models.google.GoogleTarget;
import com.serphacker.serposcope.scraper.google.GoogleDevice;
import static com.serphacker.serposcope.scraper.google.GoogleDevice.SMARTPHONE;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.zip.GZIPOutputStream;
import ninja.Context;
import ninja.Router;
import ninja.params.Param;
import ninja.params.PathParam;
import ninja.utils.ResponseStreams;
import org.apache.commons.lang3.StringEscapeUtils;
import org.slf4j.LoggerFactory;
@Singleton
public class GoogleTargetController extends GoogleController {
private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(GoogleTargetController.class);
@Inject
BaseDB baseDB;
@Inject
GoogleDB googleDB;
@Inject
Router router;
@Inject
ObjectMapper objectMapper;
private final static DateTimeFormatter YEAR_MONTH = DateTimeFormatter.ofPattern("yyyy-MM");
public static class TargetVariation {
public TargetVariation(GoogleSearch search, GoogleRank rank) {
this.search = search;
this.rank = rank;
}
public final GoogleSearch search;
public final GoogleRank rank;
}
public static class TargetRank {
public TargetRank(int now, int prev, String url) {
this.now = now;
this.prev = prev;
this.url = url;
}
public final int now;
public final int prev;
public final String url;
public String getRank() {
if (now == UNRANKED) {
return "-";
}
return Integer.toString(now);
}
public String getDiff() {
if (prev == UNRANKED && now != UNRANKED) {
return "in";
}
if (prev != UNRANKED && now == UNRANKED) {
return "out";
}
int diff = prev - now;
if (diff == 0) {
return "=";
}
if (diff > 0) {
return "+" + diff;
}
return Integer.toString(diff);
}
public String getDiffClass() {
String diff = getDiff();
switch (diff.charAt(0)) {
case '+':
case 'i':
return "plus";
case '-':
case 'o':
return "minus";
default:
return "";
}
}
public String getUrl() {
return url;
}
}
public Result target(Context context,
@PathParam("targetId") Integer targetId,
@Param("startDate") String startDateStr,
@Param("endDate") String endDateStr
) {
GoogleTarget target = getTarget(context, targetId);
List searches = context.getAttribute("searches", List.class);
Group group = context.getAttribute("group", Group.class);
Config config = baseDB.config.getConfig();
String display = context.getParameter("display", config.getDisplayGoogleTarget());
if (!Config.VALID_DISPLAY_GOOGLE_TARGET.contains(display) && !"export".equals(display)) {
display = Config.DEFAULT_DISPLAY_GOOGLE_TARGET;
}
if (target == null) {
context.getFlashScope().error("error.invalidTarget");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
Run minRun = baseDB.run.findFirst(group.getModule(), RunDB.STATUSES_DONE, null);
Run maxRun = baseDB.run.findLast(group.getModule(), RunDB.STATUSES_DONE, null);
if (maxRun == null || minRun == null || searches.isEmpty()) {
String fallbackDisplay = "export".equals(display) ? "table" : display;
return Results.ok()
.template("/serposcope/views/google/GoogleTargetController/" + fallbackDisplay + ".ftl.html")
.render("startDate", "")
.render("endDate", "")
.render("display", fallbackDisplay)
.render("target", target);
}
LocalDate minDay = minRun.getDay();
LocalDate maxDay = maxRun.getDay();
LocalDate startDate = null;
if (startDateStr != null) {
try {
startDate = LocalDate.parse(startDateStr);
} catch (Exception ex) {
}
}
LocalDate endDate = null;
if (endDateStr != null) {
try {
endDate = LocalDate.parse(endDateStr);
} catch (Exception ex) {
}
}
if (startDate == null || endDate == null || endDate.isBefore(startDate)) {
startDate = maxDay.minusDays(30);
endDate = maxDay;
}
Run firstRun = baseDB.run.findFirst(group.getModule(), RunDB.STATUSES_DONE, startDate);
Run lastRun = baseDB.run.findLast(group.getModule(), RunDB.STATUSES_DONE, endDate);
List runs = baseDB.run.listDone(firstRun.getId(), lastRun.getId());
startDate = firstRun.getDay();
endDate = lastRun.getDay();
switch (display) {
case "table":
case "variation":
return Results.ok()
.template("/serposcope/views/google/GoogleTargetController/" + display + ".ftl.html")
.render("target", target)
.render("searches", searches)
.render("startDate", startDate.toString())
.render("endDate", endDate.toString())
.render("minDate", minDay)
.render("maxDate", maxDay)
.render("display", display);
case "chart":
return renderChart(group, target, searches, runs, minDay, maxDay, startDate, endDate);
case "export":
return renderExport(group, target, searches, runs, minDay, maxDay, startDate, endDate);
default:
throw new IllegalStateException();
}
}
public Result jsonVariation(
Context context,
@PathParam("targetId") Integer targetId,
@Param("endDate") String endDateStr
) {
GoogleTarget target = getTarget(context, targetId);
List searches = context.getAttribute("searches", List.class);
Group group = context.getAttribute("group", Group.class);
final LocalDate endDate;
try {
endDate = LocalDate.parse(endDateStr);
} catch (Exception ex) {
return Results.json().renderRaw("[[],[],[]]");
}
Run lastRun = baseDB.run.findLast(group.getModule(), RunDB.STATUSES_DONE, endDate);
List ranksUp = new ArrayList<>();
List ranksDown = new ArrayList<>();
List ranksSame = new ArrayList<>();
Map searchesById = searches.stream()
.collect(Collectors.toMap(GoogleSearch::getId, Function.identity()));
List ranks = googleDB.rank.list(lastRun.getId(), group.getId(), target.getId());
for (GoogleRank rank : ranks) {
GoogleSearch search = searchesById.get(rank.googleSearchId);
if (search == null) {
continue;
}
if (rank.diff > 0) {
ranksDown.add(new TargetVariation(search, rank));
} else if (rank.diff < 0) {
ranksUp.add(new TargetVariation(search, rank));
} else {
ranksSame.add(new TargetVariation(search, rank));
}
}
Collections.sort(ranksUp, (TargetVariation o1, TargetVariation o2) -> Integer.compare(o1.rank.diff, o2.rank.diff));
Collections.sort(ranksDown, (TargetVariation o1, TargetVariation o2) -> -Integer.compare(o1.rank.diff, o2.rank.diff));
Collections.sort(ranksSame, (TargetVariation o1, TargetVariation o2) -> Integer.compare(o1.rank.rank, o2.rank.rank));
return Results.ok()
.json()
.render((Context context0, Result result) -> {
PrintWriter writer = null;
OutputStream os = null;
try {
String acceptEncoding = context0.getHeader("Accept-Encoding");
if (acceptEncoding != null && acceptEncoding.contains("gzip")) {
result.addHeader("Content-Encoding", "gzip");
}
ResponseStreams response = context0.finalizeHeaders(result);
os = response.getOutputStream();
if (acceptEncoding != null && acceptEncoding.contains("gzip")) {
os = new GZIPOutputStream(os);
}
writer = new PrintWriter(os);
writer.append("[");
int id = 0;
writer.append("[");
for (int i = 0; i < ranksUp.size(); i++) {
TargetVariation var = ranksUp.get(i);
writer
.append("{")
.append("\"id\":").append(Integer.toString(id++))
.append(",\"search\":").append(searchToJson(var.search))
.append(",\"now\":").append(Integer.toString(var.rank.rank))
.append(",\"prv\":").append(Integer.toString(var.rank.previousRank))
.append(",\"diff\":").append(Integer.toString(var.rank.diff))
.append("}");
if(i != ranksUp.size()-1){
writer.append(',');
}
}
writer.append("],[");
for (int i = 0; i < ranksDown.size(); i++) {
TargetVariation var = ranksDown.get(i);
writer
.append("{")
.append("\"id\":").append(Integer.toString(id++))
.append(",\"search\":").append(searchToJson(var.search))
.append(",\"now\":").append(Integer.toString(var.rank.rank))
.append(",\"prv\":").append(Integer.toString(var.rank.previousRank))
.append(",\"diff\":").append(Integer.toString(var.rank.diff))
.append("}");
if(i != ranksDown.size()-1){
writer.append(',');
}
}
writer.append("],[");
for (int i = 0; i < ranksSame.size(); i++) {
TargetVariation var = ranksSame.get(i);
writer
.append("{")
.append("\"id\":").append(Integer.toString(id++))
.append(",\"search\":").append(searchToJson(var.search))
.append(",\"now\":").append(Integer.toString(var.rank.rank))
.append("}");
if(i != ranksSame.size()-1){
writer.append(',');
}
}
writer.append("]]");
} catch (Exception ex) {
LOG.warn("HTTP error", ex);
} finally {
if (os != null) {
try {
writer.close();
os.close();
} catch (Exception ex) {
}
}
}
});
}
protected StringBuilder searchToJson(GoogleSearch search) {
StringBuilder searchesJson = new StringBuilder("{");
searchesJson.append("\"id\":")
.append(search.getId())
.append(",");
searchesJson.append("\"keyword\":\"")
.append(StringEscapeUtils.escapeJson(search.getKeyword()))
.append("\",");
searchesJson.append("\"country\":\"")
.append(search.getCountry().name())
.append("\",");
searchesJson.append("\"device\":\"")
.append(SMARTPHONE.equals(search.getDevice()) ? 'M' : 'D')
.append("\",");
searchesJson.append("\"local\":\"")
.append(search.getLocal() == null ? "" : StringEscapeUtils.escapeJson(search.getLocal()))
.append("\",");
searchesJson.append("\"datacenter\":\"")
.append(search.getDatacenter() == null ? "" : StringEscapeUtils.escapeJson(search.getDatacenter()))
.append("\",");
searchesJson.append("\"custom\":\"")
.append(search.getCustomParameters() == null ? "" : StringEscapeUtils.escapeJson(search.getCustomParameters()))
.append("\"");
searchesJson.append("}");
return searchesJson;
}
protected Result renderChart(
Group group,
GoogleTarget target,
List searches,
List runs,
LocalDate minDay,
LocalDate maxDay,
LocalDate startDate,
LocalDate endDate
) {
String display = "chart";
StringBuilder builder = new StringBuilder("{\"searches\": [");
for (GoogleSearch search : searches) {
builder.append("\"").append(StringEscapeUtils.escapeJson(search.getKeyword())).append("\",");
}
builder.setCharAt(builder.length() - 1, ']');
builder.append(",\"ranks\": [");
int maxRank = 0;
for (Run run : runs) {
builder.append("\n\t[").append(run.getStarted().toEpochSecond(ZoneOffset.UTC) * 1000l).append(",");
// calendar
builder.append("null,");
Map ranks = googleDB.rank.list(run.getId(), group.getId(), target.getId())
.stream().collect(Collectors.toMap((g) -> g.googleSearchId, Function.identity()));
for (GoogleSearch search : searches) {
GoogleRank fullRank = ranks.get(search.getId());
// GoogleRank fullRank = googleDB.rank.getFull(run.getId(), group.getId(), target.getId(), search.getId());
if (fullRank != null && fullRank.rank != GoogleRank.UNRANKED && fullRank.rank > maxRank) {
maxRank = fullRank.rank;
}
builder.append(fullRank == null || fullRank.rank == GoogleRank.UNRANKED ? "null" : fullRank.rank).append(',');
}
builder.setCharAt(builder.length() - 1, ']');
builder.append(",");
}
builder.setCharAt(builder.length() - 1, ']');
builder.append(",\n\"maxRank\": ").append(maxRank).append("}");
List events = baseDB.event.list(group, startDate, endDate);
String jsonEvents = null;
try {
jsonEvents = objectMapper.writeValueAsString(events);
} catch (JsonProcessingException ex) {
jsonEvents = "[]";
}
Map bestRanks = new HashMap<>();
for (GoogleSearch search : searches) {
bestRanks.put(search.getId(), googleDB.rank.getBest(target.getGroupId(), target.getId(), search.getId()));
}
return Results.ok()
.template("/serposcope/views/google/GoogleTargetController/" + display + ".ftl.html")
.render("target", target)
.render("searches", searches)
.render("startDate", startDate.toString())
.render("endDate", endDate.toString())
.render("minDate", minDay)
.render("maxDate", maxDay)
.render("display", display)
.render("ranksJson", builder.toString())
.render("eventsJson", jsonEvents);
}
protected Result renderExport(
Group group,
GoogleTarget target,
List searches,
List runs,
LocalDate minDay,
LocalDate maxDay,
LocalDate startDate,
LocalDate endDate
) {
return Results.ok()
.text()
.addHeader("Content-Disposition", "attachment; filename=\"export.csv\"")
.render((Context context, Result result) -> {
ResponseStreams stream = context.finalizeHeaders(result);
try (Writer writer = stream.getWriter()) {
writer.append("date,rank,url,target,keyword,device,country,datacenter,local,custom\n");
for (Run run : runs) {
String day = run.getDay().toString();
for (GoogleSearch search : searches) {
GoogleRank rank = googleDB.rank.getFull(run.getId(), group.getId(), target.getId(), search.getId());
writer.append(day).append(",");
if (rank != null) {
writer.append(Integer.toString(rank.rank)).append(",");
writer.append(rank.url).append(",");
} else {
writer.append(",").append(",");
}
writer.append(StringEscapeUtils.escapeCsv(target.getName())).append(",");
writer.append(StringEscapeUtils.escapeCsv(search.getKeyword())).append(",");
writer.append(search.getDevice() == GoogleDevice.DESKTOP ? "D" : "M").append(",");
writer.append(search.getCountry().name()).append(",");
writer.append(
search.getDatacenter() != null
? StringEscapeUtils.escapeCsv(search.getDatacenter())
: ""
).append(",");
writer.append(
search.getLocal() != null
? StringEscapeUtils.escapeCsv(search.getLocal())
: ""
).append(",");
writer.append(
search.getCustomParameters() != null
? StringEscapeUtils.escapeCsv(search.getCustomParameters())
: ""
);
writer.append("\n");
}
}
} catch (IOException ex) {
LOG.warn("error while exporting csv");
}
});
}
public Result jsonRanks(
Context context,
@PathParam("targetId") Integer targetId,
@Param("startDate") String startDateStr,
@Param("endDate") String endDateStr
) {
final GoogleTarget target = getTarget(context, targetId);
final List searches = context.getAttribute("searches", List.class);
final Group group = context.getAttribute("group", Group.class);
final LocalDate startDate, endDate;
try {
startDate = LocalDate.parse(startDateStr);
endDate = LocalDate.parse(endDateStr);
} catch (Exception ex) {
return Results.json().renderRaw("[]");
}
final Run firstRun = baseDB.run.findFirst(group.getModule(), RunDB.STATUSES_DONE, startDate);
final Run lastRun = baseDB.run.findLast(group.getModule(), RunDB.STATUSES_DONE, endDate);
final List runs = baseDB.run.listDone(firstRun.getId(), lastRun.getId());
return Results.ok()
.json()
.render((Context context0, Result result) -> {
PrintWriter writer = null;
OutputStream os = null;
try {
String acceptEncoding = context0.getHeader("Accept-Encoding");
if (acceptEncoding != null && acceptEncoding.contains("gzip")) {
result.addHeader("Content-Encoding", "gzip");
}
ResponseStreams response = context0.finalizeHeaders(result);
os = response.getOutputStream();
if (acceptEncoding != null && acceptEncoding.contains("gzip")) {
os = new GZIPOutputStream(os);
}
writer = new PrintWriter(os);
getTableJson(group, target, searches, runs, startDate, endDate, writer);
} catch (Exception ex) {
LOG.warn("HTTP error", ex);
} finally {
if (os != null) {
try {
writer.close();
os.close();
} catch (Exception ex) {
}
}
}
});
}
protected void getTableJson(
Group group,
GoogleTarget target,
List searches,
List runs,
LocalDate startDate,
LocalDate endDate,
Writer writer
) throws IOException {
writer.append("[[[-1, 0, 0, [");
if (runs.isEmpty() || searches.isEmpty()) {
writer.append("]]],[]]");
}
// events
List events = baseDB.event.list(group, startDate, endDate);
for (int i = 0; i < runs.size(); i++) {
Run run = runs.get(i);
Event event = null;
for (Event candidat : events) {
if (run.getDay().equals(candidat.getDay())) {
event = candidat;
break;
}
}
if (event != null) {
writer
.append("[\"").append(StringEscapeUtils.escapeJson(event.getTitle())).append("\",")
.append('"').append(StringEscapeUtils.escapeJson(event.getDescription())).append("\"]");
} else {
writer.append("0");
}
if (i != runs.size() - 1) {
writer.append(",");
}
}
writer.append("]],");
Map builders = new HashMap<>();
for (GoogleSearch search : searches) {
StringBuilder builder;
builders.put(search.getId(), builder = new StringBuilder("["));
GoogleBest best = googleDB.rank.getBest(target.getGroupId(), target.getId(), search.getId());
builder
.append(search.getId())
.append(",[\"").append(StringEscapeUtils.escapeJson(search.getKeyword()))
.append("\",\"").append(search.getCountry().name())
.append("\",\"").append(SMARTPHONE.equals(search.getDevice()) ? 'M' : 'D')
.append("\",\"").append(search.getLocal() == null ? "" : StringEscapeUtils.escapeJson(search.getLocal()))
.append("\",\"").append(search.getDatacenter() == null ? "" : StringEscapeUtils.escapeJson(search.getDatacenter()))
.append("\",\"").append(search.getCustomParameters() == null ? "" : StringEscapeUtils.escapeJson(search.getCustomParameters()))
.append("\"],");
if (best == null) {
builder.append("0,");
} else {
builder
.append("[").append(best.getRank())
.append(",\"").append(best.getRunDay() != null ? best.getRunDay().toLocalDate().toString() : "?")
.append("\",\"").append(StringEscapeUtils.escapeJson(best.getUrl()))
.append("\"],");
}
builder.append("[");
}
for (int i = 0; i < runs.size(); i++) {
Run run = runs.get(i);
Map ranks = googleDB.rank.list0(run.getId(), group.getId(), target.getId())
.stream().collect(Collectors.toMap((r) -> r.googleSearchId, Function.identity()));
for (GoogleSearch search : searches) {
StringBuilder builder = builders.get(search.getId());
GoogleRank fullRank = ranks.get(search.getId());
if (fullRank != null && fullRank.rank != GoogleRank.UNRANKED) {
builder.append("[").append(fullRank.rank)
.append(",").append(fullRank.previousRank)
.append(",\"").append(StringEscapeUtils.escapeJson(fullRank.url))
.append("\"],");
} else {
builder.append("0,");
}
if (i == runs.size() - 1) {
builder.deleteCharAt(builder.length() - 1);
builder.append("]]");
}
}
}
List buildersArray = new ArrayList<>(builders.values());
for (int i = 0; i < buildersArray.size(); i++) {
writer.append(buildersArray.get(i));
if (i != buildersArray.size() - 1) {
writer.append(",");
}
}
writer.append("],[");
for (int i = 0; i < runs.size(); i++) {
Run run = runs.get(i);
writer.append("\"").append(run.getDay().toString()).append("\"");
if (i != runs.size() - 1) {
writer.append(",");
}
}
writer.append("]]");
}
protected String getTableJsonData0(
Group group,
GoogleTarget target,
List searches,
List runs,
LocalDate startDate,
LocalDate endDate
) {
StringBuilder jsonData = new StringBuilder("[{\"id\": -1, \"best\": null, \"days\": [");
if (runs.isEmpty() || searches.isEmpty()) {
jsonData.append("]}]");
return jsonData.toString();
}
// events
List events = baseDB.event.list(group, startDate, endDate);
for (Run run : runs) {
Event event = null;
for (Event candidat : events) {
if (run.getDay().equals(candidat.getDay())) {
event = candidat;
break;
}
}
if (event != null) {
jsonData
.append("{\"title\":\"").append(StringEscapeUtils.escapeJson(event.getTitle()))
.append("\",\"description\":\"").append(StringEscapeUtils.escapeJson(event.getDescription()))
.append("\"},");
} else {
jsonData.append("null,");
}
}
jsonData.deleteCharAt(jsonData.length() - 1);
jsonData.append("]},");
Map builders = new HashMap<>();
for (GoogleSearch search : searches) {
StringBuilder builder;
builders.put(search.getId(), builder = new StringBuilder());
builder.append("");
GoogleBest best = googleDB.rank.getBest(target.getGroupId(), target.getId(), search.getId());
builder.append("{\"id\":").append(search.getId())
.append(",\"search\":{")
.append("\"id\":").append(search.getId())
.append(",\"k\":\"").append(StringEscapeUtils.escapeJson(search.getKeyword()))
.append("\",\"t\":\"").append(search.getCountry().name())
.append("\",\"d\":\"").append(SMARTPHONE.equals(search.getDevice()) ? 'M' : 'D')
.append("\",\"l\":\"").append(search.getLocal() == null ? "" : StringEscapeUtils.escapeJson(search.getLocal()))
.append("\",\"dc\":\"").append(search.getDatacenter() == null ? "" : StringEscapeUtils.escapeJson(search.getDatacenter()))
.append("\",\"c\":\"").append(search.getCustomParameters() == null ? "" : StringEscapeUtils.escapeJson(search.getCustomParameters()))
.append("\"}, \"best\":");
if (best == null) {
builder.append("null,");
} else {
builder
.append("{\"rank\":").append(best.getRank())
.append(",\"date\":\"").append(best.getRunDay() != null ? best.getRunDay().toLocalDate().toString() : "?")
.append("\",\"url\":\"").append(StringEscapeUtils.escapeJson(best.getUrl()))
.append("\"},");
}
builder.append("\"days\": [");
}
for (int i = 0; i < runs.size(); i++) {
Run run = runs.get(i);
Map ranks = googleDB.rank.list0(run.getId(), group.getId(), target.getId())
.stream().collect(Collectors.toMap((r) -> r.googleSearchId, Function.identity()));
for (GoogleSearch search : searches) {
StringBuilder builder = builders.get(search.getId());
GoogleRank fullRank = ranks.get(search.getId());
if (fullRank != null && fullRank.rank != GoogleRank.UNRANKED) {
builder.append("{\"r\":").append(fullRank.rank)
.append(",\"p\":").append(fullRank.previousRank)
.append(",\"u\":\"").append(StringEscapeUtils.escapeJson(fullRank.url))
.append("\"},");
} else {
builder.append("{\"r\":32767,\"p\":null,\"u\":null},");
}
if (i == runs.size() - 1) {
builder.deleteCharAt(builder.length() - 1);
builder.append("]},");
}
}
}
for (StringBuilder value : builders.values()) {
jsonData.append(value);
}
jsonData.deleteCharAt(jsonData.length() - 1);
jsonData.append("]");
return jsonData.toString();
}
}