serposcope.controllers.google.GoogleGroupController Maven / Gradle / Ivy
/*
* Serposcope - SEO rank checker https://serposcope.serphacker.com/
*
* Copyright (c) 2016 SERP Hacker
* @author Pierre Nogues
* @license https://opensource.org/licenses/MIT MIT License
*/
package serposcope.controllers.google;
import com.google.common.base.Optional;
import com.google.inject.Inject;
import ninja.Result;
import ninja.Results;
import com.google.inject.Singleton;
import com.serphacker.serposcope.db.base.BaseDB;
import com.serphacker.serposcope.db.base.RunDB;
import com.serphacker.serposcope.db.google.GoogleDB;
import com.serphacker.serposcope.models.base.Event;
import com.serphacker.serposcope.models.base.Group;
import com.serphacker.serposcope.models.base.Run;
import com.serphacker.serposcope.models.google.GoogleSearch;
import com.serphacker.serposcope.models.google.GoogleTarget;
import com.serphacker.serposcope.models.google.GoogleTarget.PatternType;
import com.serphacker.serposcope.models.google.GoogleTargetSummary;
import com.serphacker.serposcope.scraper.google.GoogleCountryCode;
import com.serphacker.serposcope.scraper.google.GoogleDevice;
import static com.serphacker.serposcope.scraper.google.GoogleDevice.SMARTPHONE;
import com.serphacker.serposcope.task.TaskManager;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.IDN;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.GZIPOutputStream;
import ninja.Context;
import ninja.FilterWith;
import ninja.Router;
import ninja.i18n.Messages;
import ninja.params.Param;
import ninja.params.Params;
import ninja.session.FlashScope;
import ninja.utils.ResponseStreams;
import org.apache.commons.lang3.StringEscapeUtils;
import org.jsoup.Jsoup;
import org.jsoup.safety.Whitelist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import serposcope.controllers.GroupController;
import serposcope.filters.AdminFilter;
import serposcope.filters.XSRFFilter;
import serposcope.helpers.Validator;
@Singleton
public class GoogleGroupController extends GoogleController {
private static final Logger LOG = LoggerFactory.getLogger(GoogleGroupController.class);
@Inject
Router router;
@Inject
BaseDB baseDB;
@Inject
GoogleDB googleDB;
@Inject
TaskManager taskManager;
@Inject
Messages msg;
final Object searchLock = new Object();
public Result view(Context context) {
Group group = context.getAttribute("group", Group.class);
Map summaryByTagetId = new HashMap<>();
Map> scoreHistoryByTagetId = new HashMap<>();
Run lastRun = baseDB.run.findLast(group.getModule(), RunDB.STATUSES_DONE, null);
if (lastRun != null) {
List summaries = googleDB.targetSummary.list(lastRun.getId());
for (GoogleTargetSummary summary : summaries) {
if (summary != null) {
summaryByTagetId.put(summary.getTargetId(), summary);
List scoreHistory = googleDB.targetSummary.listScoreHistory(group.getId(), summary.getTargetId(), 30);
int missingScore = 30 - scoreHistory.size();
for (int i = 0; i < missingScore; i++) {
scoreHistory.add(0, 0);
}
scoreHistoryByTagetId.put(summary.getTargetId(), scoreHistory);
}
}
}
return Results
.ok()
.render("events", baseDB.event.list(group, null, null))
.render("default", googleDB.options.get())
.render("searchesSize", context.getAttribute("searches", List.class).size())
.render("targets", context.getAttribute("targets"))
.render("summaries", summaryByTagetId)
.render("histories", scoreHistoryByTagetId);
}
public Result jsonSearches(Context context) {
List searches = context.getAttribute("searches", List.class);
if (searches.isEmpty()) {
return Results.json().renderRaw("[]");
}
return Results.ok()
.json()
.render((Context context0, Result result) -> {
PrintWriter writer = null;
OutputStream os = null;
try {
String acceptEncoding = context0.getHeader("Accept-Encoding");
if (acceptEncoding != null && acceptEncoding.contains("gzip")) {
result.addHeader("Content-Encoding", "gzip");
}
ResponseStreams response = context0.finalizeHeaders(result);
os = response.getOutputStream();
if (acceptEncoding != null && acceptEncoding.contains("gzip")) {
os = new GZIPOutputStream(os);
}
writer = new PrintWriter(os);
writer.append("[");
for (int i = 0; i < searches.size(); i++) {
GoogleSearch search = searches.get(i);
writer.append("{");
writer.append("\"id\":")
.append(Integer.toString(search.getId()))
.append(",");
writer.append("\"keyword\":\"")
.append(StringEscapeUtils.escapeJson(search.getKeyword()))
.append("\",");
writer.append("\"country\":\"")
.append(search.getCountry().name())
.append("\",");
writer.append("\"device\":\"")
.append(SMARTPHONE.equals(search.getDevice()) ? 'M' : 'D')
.append("\",");
writer.append("\"local\":\"")
.append(search.getLocal() == null ? "" : StringEscapeUtils.escapeJson(search.getLocal()))
.append("\",");
writer.append("\"datacenter\":\"")
.append(search.getDatacenter() == null ? "" : StringEscapeUtils.escapeJson(search.getDatacenter()))
.append("\",");
writer.append("\"custom\":\"")
.append(search.getCustomParameters() == null ? "" : StringEscapeUtils.escapeJson(search.getCustomParameters()))
.append("\"");
writer.append("}");
if(i != searches.size()-1){
writer.append(",");
}
}
writer.append("]");
} catch (Exception ex) {
LOG.warn("HTTP error", ex);
} finally {
if (os != null) {
try {
writer.close();
os.close();
} catch (Exception ex) {
}
}
}
});
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result addSearch(
Context context,
@Params("keyword[]") String[] keywords,
@Params("country[]") String country[], @Params("datacenter[]") String[] datacenters,
@Params("device[]") Integer[] devices,
@Params("local[]") String[] locals, @Params("custom[]") String[] customs
) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
if (keywords == null || country == null || datacenters == null || devices == null || locals == null || customs == null
|| keywords.length != country.length || keywords.length != datacenters.length || keywords.length != devices.length
|| keywords.length != locals.length || keywords.length != customs.length) {
flash.error("error.invalidParameters");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
Set searches = new HashSet<>();
for (int i = 0; i < keywords.length; i++) {
GoogleSearch search = new GoogleSearch();
if (keywords[i].isEmpty()) {
flash.error("admin.google.keywordEmpty");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
search.setKeyword(keywords[i]);
GoogleCountryCode countryCode = null;
if(country[i] != null){
try {
countryCode = GoogleCountryCode.valueOf(country[i].toUpperCase());
} catch(Exception ex){
}
}
if (countryCode == null) {
flash.error("admin.google.invalidCountry");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
search.setCountry(countryCode);
if (!datacenters[i].isEmpty()) {
if (!Validator.isIPv4(datacenters[i])) {
flash.error("error.invalidIP");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
search.setDatacenter(datacenters[i]);
}
if (devices[i] != null && devices[i] >= 0 && devices[i] < GoogleDevice.values().length) {
search.setDevice(GoogleDevice.values()[devices[i]]);
} else {
search.setDevice(GoogleDevice.DESKTOP);
}
if (!Validator.isEmpty(locals[i])) {
search.setLocal(locals[i]);
}
if (!Validator.isEmpty(customs[i])) {
search.setCustomParameters(customs[i]);
}
searches.add(search);
}
List knownSearches = new ArrayList<>();
synchronized (searchLock) {
for (GoogleSearch search : searches) {
int id = googleDB.search.getId(search);
if (id > 0) {
search.setId(id);
knownSearches.add(search);
}
}
googleDB.search.insert(searches, group.getId());
}
googleDB.serpRescan.rescan(null, getTargets(context), knownSearches, false);
flash.success("google.group.searchInserted");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()) + "#tab-searches");
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result addTarget(
Context context,
@Param("target-radio") String targetType,
@Params("name[]") String[] names,
@Params("pattern[]") String[] patterns
) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
if (targetType == null
|| names == null || names.length == 0
|| patterns == null || patterns.length == 0
|| names.length != patterns.length) {
flash.error("error.invalidParameters");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
Set targets = new HashSet<>();
for (int i = 0; i < names.length; i++) {
String name = names[i];
String pattern = patterns[i];
if (name != null) {
name = name.replaceAll("(^\\s+)|(\\s+$)", "");
}
if (pattern != null) {
pattern = pattern.replaceAll("(^\\s+)|(\\s+$)", "");
}
if (Validator.isEmpty(name)) {
flash.error("error.invalidName");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
PatternType type = null;
try {
type = PatternType.valueOf(targetType);
} catch (Exception ex) {
flash.error("error.invalidTargetType");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
if(PatternType.DOMAIN.equals(type) || PatternType.SUBDOMAIN.equals(type)){
try {
pattern = IDN.toASCII(pattern);
} catch(Exception ex) {
pattern = null;
}
}
if (!GoogleTarget.isValidPattern(type, pattern)) {
flash.error("error.invalidPattern");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
targets.add(new GoogleTarget(group.getId(), name, type, pattern));
}
if (googleDB.target.insert(targets) < 1) {
flash.error("error.internalError");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
googleDB.serpRescan.rescan(null, targets, getSearches(context), true);
Run runningGoogleTask = taskManager.getRunningGoogleTask();
if (runningGoogleTask != null) {
flash.put("warning", msg.get("google.group.websiteInsertedWhileRun", context, Optional.absent(), runningGoogleTask.getId()).or(""));
} else {
flash.success("google.group.websiteInserted");
}
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result renameTarget(
Context context,
@Param("name") String name,
@Param("id") Integer targetId) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
GoogleTarget target = getTarget(context, targetId);
if (target == null) {
flash.error("error.invalidWebsite");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
if (name != null) {
name = name.replaceAll("(^\\s+)|(\\s+$)", "");
}
if (Validator.isEmpty(name)) {
flash.error("error.invalidName");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
target.setName(name);
googleDB.target.rename(target);
flash.success("google.group.websiteRenamed");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result delSearch(
Context context,
@Params("id[]") String[] ids
) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
if (ids == null || ids.length == 0) {
flash.error("error.noSearchSelected");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
List searches = new ArrayList<>();
for (String id : ids) {
GoogleSearch search = null;
try {
search = getSearch(context, Integer.parseInt(id));
} catch (Exception ex) {
search = null;
}
if (search == null) {
flash.error("error.invalidSearch");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
searches.add(search);
}
// TODO FIX ME locking until database modification done
if (taskManager.isGoogleRunning()) {
flash.error("admin.google.errorTaskRunning");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
for (GoogleSearch search : searches) {
deleteSearch(group, search);
}
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()) + "#tab-searches");
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result delTarget(
Context context,
@Params("id[]") String[] ids
) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
// TODO FIX ME locking until database modification done
if (taskManager.isGoogleRunning()) {
flash.error("admin.google.errorTaskRunning");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
if (ids == null || ids.length == 0) {
flash.error("error.noWebsiteSelected");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
for (String id : ids) {
GoogleTarget target = null;
try {
target = getTarget(context, Integer.parseInt(id));
} catch (Exception ex) {
target = null;
}
if (target == null) {
flash.error("error.invalidWebsite");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
googleDB.targetSummary.deleteByTarget(target.getId());
googleDB.rank.deleteByTarget(group.getId(), target.getId());
googleDB.target.delete(target.getId());
}
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result delete(Context context) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
// TODO FIX ME locking until database modification done
if (taskManager.isGoogleRunning()) {
flash.error("admin.google.errorTaskRunning");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
List targets = googleDB.target.list(Arrays.asList(group.getId()));
for (GoogleTarget target : targets) {
googleDB.targetSummary.deleteByTarget(target.getId());
googleDB.rank.deleteByTarget(group.getId(), target.getId());
googleDB.target.delete(target.getId());
}
List searches = googleDB.search.listByGroup(Arrays.asList(group.getId()));
for (GoogleSearch search : searches) {
deleteSearch(group, search);
}
baseDB.event.delete(group);
baseDB.user.delPerm(group);
if (!baseDB.group.delete(group)) {
flash.error("admin.google.failedDeleteGroup");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
} else {
flash.success("admin.google.groupDeleted");
return Results.redirect(router.getReverseRoute(GroupController.class, "groups"));
}
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result exportSearches(
Context context,
@Params("id[]") String[] ids
) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
if (ids == null || ids.length == 0) {
flash.error("error.noSearchSelected");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
List searches = new ArrayList<>();
for (String id : ids) {
GoogleSearch search = null;
try {
search = getSearch(context, Integer.parseInt(id));
} catch (Exception ex) {
search = null;
}
if (search == null) {
flash.error("error.invalidSearch");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
searches.add(search);
}
StringBuilder builder = new StringBuilder();
for (GoogleSearch search : searches) {
builder.append(StringEscapeUtils.escapeCsv(search.getKeyword())).append(",");
builder.append(search.getCountry()).append(",");
builder.append(search.getDatacenter() != null ? search.getDatacenter() : "").append(",");
builder.append(search.getDevice() != null ? (search.getDevice() == GoogleDevice.DESKTOP ? "desktop" : "mobile") : "").append(",");
builder.append(StringEscapeUtils.escapeCsv(search.getLocal() != null ? search.getLocal() : "")).append(",");
builder.append(StringEscapeUtils.escapeCsv(search.getCustomParameters() != null ? search.getCustomParameters() : "")).append("\n");
}
return Results.ok().text().render(builder.toString());
}
protected void deleteSearch(Group group, GoogleSearch search) {
synchronized (searchLock) {
googleDB.search.deleteFromGroup(search, group.getId());
googleDB.rank.deleteBySearch(group.getId(), search.getId());
if (!googleDB.search.hasGroup(search)) {
googleDB.serp.deleteBySearch(search.getId());
googleDB.search.delete(search);
}
}
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result addEvent(Context context,
@Param("day") String day,
@Param("title") String title,
@Param("description") String description,
@Param("redir-search") Integer redirSearchId,
@Param("redir-target") Integer redirTargetId
) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
Event event = new Event();
event.setGroupId(group.getId());
try {
event.setDay(LocalDate.parse(day));
} catch (Exception ex) {
}
if (event.getDay() == null) {
flash.error("error.invalidDate");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
if (Validator.isEmpty(title)) {
flash.error("error.invalidTitle");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
if (baseDB.event.find(group, event.getDay()) != null) {
flash.error("google.group.alreadyEventForThisDate");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
event.setTitle(title);
event.setDescription(Jsoup.clean(description == null ? "" : description, Whitelist.basic()));
if (!baseDB.event.insert(event)) {
flash.error("error.internalError");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
flash.success("google.group.eventInserted");
if (redirSearchId != null) {
return Results.redirect(router.getReverseRoute(GoogleSearchController.class, "search", "groupId", group.getId(),
"searchId", redirSearchId));
}
if (redirTargetId != null) {
return Results.redirect(router.getReverseRoute(GoogleTargetController.class, "target", "groupId", group.getId(),
"targetId", redirTargetId));
}
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result delEvent(Context context, @Param("day") String day) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
Event event = new Event();
event.setGroupId(group.getId());
try {
event.setDay(LocalDate.parse(day));
} catch (Exception ex) {
}
if (event.getDay() == null) {
flash.error("error.invalidDate");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
baseDB.event.delete(event);
flash.success("google.group.eventDeleted");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
@FilterWith({
XSRFFilter.class,
AdminFilter.class
})
public Result rename(Context context, @Param("name") String name) {
FlashScope flash = context.getFlashScope();
Group group = context.getAttribute("group", Group.class);
if (Validator.isEmpty(name)) {
flash.error("error.invalidName");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
group.setName(name);
baseDB.group.update(group);
flash.success("google.group.groupRenamed");
return Results.redirect(router.getReverseRoute(GoogleGroupController.class, "view", "groupId", group.getId()));
}
public Result jsonTargetSuggest(
Context context,
@Param("query") String query
) {
StringBuilder builder = new StringBuilder("[");
getTargets(context).stream()
.filter((g) -> query == null ? true : g.getName().contains(query))
.sorted((o1, o2) -> o1.getId() - o2.getId())
.limit(10)
.forEach((g) -> {
builder.append("{")
.append("\"id\":").append(g.getId()).append(",")
.append("\"name\":\"").append(StringEscapeUtils.escapeJson(g.getName())).append("\",")
.append("\"group\":").append(g.getGroupId())
.append("},");
});
if (builder.length() > 1) {
builder.deleteCharAt(builder.length() - 1);
}
builder.append("]");
return Results.json().renderRaw(builder.toString());
}
public Result jsonSearchSuggest(
Context context,
@Param("query") String query
) {
StringBuilder builder = new StringBuilder("[");
getSearches(context).stream()
.filter((g) -> query == null ? true : g.getKeyword().contains(query))
.sorted((o1, o2) -> o1.getId() - o2.getId())
.limit(10)
.forEach((g) -> {
builder.append("{")
.append("\"id\":").append(g.getId()).append(",")
.append("\"name\":\"").append(StringEscapeUtils.escapeJson(g.getKeyword())).append("\"")
.append("},");
});
if (builder.length() > 1) {
builder.deleteCharAt(builder.length() - 1);
}
builder.append("]");
return Results.json().renderRaw(builder.toString());
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy