环境
- 前端：html,css,js,jQuery,bootstrap
- 后端：flask
- 搜索引擎：elasticsearch
- 数据源：某某之家
项目展示
项目目录
主要源码
获取数据源并写入es
from lxml import etree
from concurrent.futures import ThreadPoolExecutor
from elasticsearch import Elasticsearch
from elasticsearch import helpers
import requests
headers = {
'user-agent': 'ua'
}
es = Elasticsearch()
if not es.indices.exists(index='car'):
es.indices.create(index='car', mappings={
'properties': {
'url': {
'type': 'text'
},
'img': {
'type': 'text'
},
'title': {
'type': 'text'
},
'desc': {
'type': 'text'
}
}
})
def task(url,page):
res = requests.get(url, headers)
text = res.text
tree = etree.HTML(text)
ul_list = tree.xpath('//ul[@class="article"]')
actions = []
for ul in ul_list:
li_list = ul.xpath('./li')
for li in li_list:
url = li.xpath('./a/@href'),
img = li.xpath('./a/div/img/@src'),
desc = li.xpath('./a/p/text()'),
title = li.xpath('./a/h3/text()')
if title:
doc = {
'_index': 'car',
'url': f'https:{url[0][0]}',
'img': img[0][0],
'desc': desc[0][0],
'title': title[0],
}
actions.append(doc)
helpers.bulk(es, actions=actions)
print(f'第{page}页完成！')
def main():
with ThreadPoolExecutor() as pool:
for i in range(1, 11):
url = f'https://www.autohome.com.cn/all/{i}/'
pool.submit(task, url=url,page=i)
if __name__ == '__main__':
main()
视图函数
from flask import Blueprint
from flask import request
from flask import render_template
from flask import jsonify
from web.ext import es
from pprint import pprint
search_bp = Blueprint('search', __name__, url_prefix='/search')
@search_bp.route('/', methods=['get', 'post'])
def search():
if request.method == 'GET':
return render_template('search.html')
elif request.method == 'POST':
content = request.values.get('content')
size = 10
current = int(request.values.get('current', '0'))
if content:
res = es.search(index='car', query={
'match': {
"title": content
}
}, highlight={
"pre_tags": "<span style='color: red'>"
,
"post_tags": "</span>"
,
"fields": {
"title": {}
}
}, size=1000)
else:
res = es.search(index='car', query={
'match_all': {}
}, size=1000)
new_res = res['hits']['hits']
total = int(res['hits']['total']['value'])
need_page = (total // size) + 1
data = {
'res': new_res[current * size:current * size + size],
'need_page': need_page,
'total': total
}
return jsonify(data)
前端
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>General Search</title>
<link rel="stylesheet" href="{{ url_for('static',filename='css/bootstrap.min.css') }}">
<script src="{{ url_for('static',filename='js/jQuery.js') }}"></script>
<script src="{{ url_for('static',filename='js/bootstrap.min.js') }}"></script>
<style>
.title{
font-size: 25px;
font-weight: 10;
}
.result{
color: red;
}
</style>
</head>
<body>
<div>
<nav class="navbar navbar-default">
<div class="container-fluid">
<!-- Brand and toggle get grouped for better mobile display -->
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="#" style="margin-left: 100px">General Search</a>
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
<ul class="nav navbar-nav">
</ul>
<div class="navbar-form navbar-left">
<div class="form-group">
<input type="text" class="form-control" placeholder="" id="content" style="width: 800px;margin-left: 200px">
</div>
</div>
<button class="btn btn-primary pull-right" id="submit" style="margin-top: 8px;width: 100px;">搜 索</button>
</div><!-- /.navbar-collapse -->
</div><!-- /.container-fluid -->
</nav>
</div>
<div class="container">
General为您找到相关结果约<span id='count' class="result">0</span>个
<hr>
</div>
<div class="container" id="tags">
</div>
<div class="text-center">
<nav aria-label="Page navigation">
<ul class="pagination" id="page">
</ul>
</nav>
</div>
<script>
function getData(current)
{
let content=$('#content').val()
let tags=$('#tags')
$('#count').text()
tags.empty()
$.ajax({
url:"",
type:'post',
data:{
'content':content,
'current':current
},
dataType:'JSON',
success:function (res){
let length=res.total
let need_page=res.need_page
$('#count').text(length)
$.each(res.res,function (index,value){
let source=value._source
let title=source.title
let highlight=value.highlight
if (highlight)
{
title=highlight.title
}
let div=`
<div>
<p><a href="${source.url}" target="_blank">
<span class="title">${title}</span>
<br>
<img src="${source.img}" alt="">
</a></p>
<p>${source.desc}</p>
<hr>
</div>
`
tags.append(div)
})
$('#page').empty()
for(let i=0;i<need_page;i++)
{
let tmp=i+1
if (current==i)
{
$('#page').append('<li class="active"><span class="changePage">'+tmp+'</span></li>')
}
else {
$('#page').append('<li><span class="changePage">'+tmp+'</span></li>')
}
}
}
})
}
$('#submit').click(function (){
getData(0)
})
$('#page').on('click','li',function (){
getData($(this).text()-1)
})
</script>
</body>
</html>
app配置
from flask import Flask
from flask_session import Session
from web.ext import db
from .search.search import search_bp
from flask_script import Manager
from flask_migrate import Migrate, MigrateCommand
def create_app():
app = Flask(__name__)
app.config.from_object('settings.DevelopmentConfig')
app.register_blueprint(search_bp)
Session(app)
db.init_app(app)
app = Manager(app)
Migrate(app, db)
app.add_command('db', MigrateCommand)
return app
总结
对比django,flask最后选用自由度较大的flask。
集合flask-script,flask_sqlalchemy,flask-migrate加快开发。
写一个小demo深化了对es接口的理解。
