Skip to content

I
intel_promotion_api
Commit e2aa0589 authored by 盖献康's avatar 盖献康
Browse files
Options

根据字典判断是否开启定时爬虫

parent aae54a19
Hide whitespace changes
Inline Side-by-side
Showing with 164 additions and 22 deletions
.idea/uiDesigner.xml 0 → 100644
View file @ e2aa0589
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.svg" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.svg" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.svg" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.svg" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>
\ No newline at end of file
src/main/java/org/rcisoft/business/crawler/scheduled/ScheduledTasks.java
View file @ e2aa0589
package org.rcisoft.business.crawler.scheduled;
import org.rcisoft.business.crawler.service.CrawlerService;
import org.rcisoft.sys.dictionary.dto.DictDataDTO;
import org.rcisoft.sys.dictionary.entity.DictData;
import org.rcisoft.sys.dictionary.service.DictDataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationListener;
......@@ -8,6 +11,8 @@ import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 爬虫 定时任务
......@@ -19,6 +24,9 @@ public class ScheduledTasks implements ApplicationListener<ContextRefreshedEvent
@Autowired
private CrawlerService crawlerService;
@Autowired
private DictDataService dictDataService;
@Value("${cron.expression}")
private String cronExpression;
......@@ -27,8 +35,15 @@ public class ScheduledTasks implements ApplicationListener<ContextRefreshedEvent
*/
@Scheduled(cron = "#{@scheduledTasks.getCronExpression()}")
public void crawlerTask() {
System.out.println("Task performed on: " + new java.util.Date());
crawlerService.startCrawler();
// 根据字典判断是否开启定时爬虫
DictDataDTO data = new DictDataDTO();
data.setDictType("timing_crawler");
List<DictData> dataList = dictDataService.findAll(data);
if (dataList.size() > 0 && "1".equals(dataList.get(0).getFlag())) {
System.out.println("Task performed on: " + new java.util.Date());
crawlerService.startCrawler();
}
}
/**
......
src/main/java/org/rcisoft/business/crawler/service/CrawlerService.java
View file @ e2aa0589
......@@ -58,39 +58,42 @@ public class CrawlerService {
}
for (String item : list) {
String articleContent = "";
String articleType = "";
String articleUrl = Jsoup.parse(item).getElementsByTag("a").attr("href");
Document doc = Jsoup.connect(articleUrl).get();
if (articleUrl.contains(MP_WEIXIN_QQ_COM)) {
String a = getWeiXin(doc);
System.out.println("weixin--" + a);
articleContent = getWeiXin(doc);
articleType = MP_WEIXIN_QQ_COM;
} else if (articleUrl.contains(CS_COM_CN)) {
String b = getCsComCn(doc);
System.out.println("cs--" + b);
articleContent = getCsComCn(doc);
articleType = CS_COM_CN;
} else if (articleUrl.contains(CBIMC_CN)) {
String c = getCbimcCn(doc);
System.out.println("cbimc--" + c);
articleContent = getCbimcCn(doc);
articleType = CBIMC_CN;
} else if (articleUrl.contains(NEWS_CN)) {
String d = getNewsCn(doc);
System.out.println("newscn--" + d);
articleContent = getNewsCn(doc);
articleType = NEWS_CN;
} else if (articleUrl.contains(E_CHINALIFE_COM)) {
String e = getChinalife(doc);
System.out.println("chinalife--" + e);
articleContent = getChinalife(doc);
articleType = E_CHINALIFE_COM;
} else if (articleUrl.contains(PROPERTY_PICC_COM)) {
String f = getPiccCom(doc);
System.out.println("picccom--" + f);
articleContent = getPiccCom(doc);
articleType = PROPERTY_PICC_COM;
} else if (articleUrl.contains(FINANCE_CNR_CN)) {
String g = getFinanceCnr(doc);
System.out.println("financecnr--" + g);
articleContent = getFinanceCnr(doc);
articleType = FINANCE_CNR_CN;
} else if (articleUrl.contains(HSBCINSURANCE_COM_CN)) {
String h = getHsbcinsurance(doc);
System.out.println("hsbcinsurance--" + h);
articleContent = getHsbcinsurance(doc);
articleType = HSBCINSURANCE_COM_CN;
} else if (articleUrl.contains(FINANCE_PEOPLE_COM_CN)) {
String i = getFinancePeople(doc);
System.out.println("financepeople--" + i);
articleContent = getFinancePeople(doc);
articleType = FINANCE_PEOPLE_COM_CN;
} else if (articleUrl.contains(FINANCE_CHINA_COM_CN)) {
String j = getFinanceChina(doc);
System.out.println("financechina--" + j);
articleContent = getFinanceChina(doc);
articleType = FINANCE_CHINA_COM_CN;
}
System.out.println(articleType + "--" + articleContent);
}
System.out.println(list.size());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment