Upload files

App/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+
+# Register your models here.
+from .models import *
+
+admin.site.register(Standards)
+admin.site.register(Level)
+admin.site.register(SearchResults)

App/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class AppConfig(AppConfig):
+    name = 'App'

App/filters.py

@@ -0,0 +1,18 @@
+from attr import fields
+import django_filters
+
+from .models import *
+
+levels = Level.objects.all().values_list('levelNumber','levelName')
+standards = Standards.objects.all().values_list('standardTitle','standardTitle')
+
+class StandardsFilter(django_filters.FilterSet):
+    # level = django_filters.ModelChoiceFilter(queryset = SearchResults.objects.values('level'))
+    level = django_filters.MultipleChoiceFilter(choices=levels)
+    title = django_filters.MultipleChoiceFilter(choices=standards)
+    class Meta:
+        model = SearchResults
+        fields = ['level','title']
+        # exclude = ['index','']
+
+        

App/forms.py

@@ -0,0 +1,11 @@
+from django import forms
+
+class searchForm(forms.Form):
+    phrase = forms.CharField(
+        max_length = 200,
+        widget= forms.TextInput(attrs={
+            "class": "search_form",
+            "placeholder": "Serach IDDRS...."
+        })
+    )
+

App/migrations/0001_initial.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.0.4 on 2022-05-23 13:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='standards',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('standardTitle', models.CharField(max_length=200)),
+                ('stamdardNumber', models.CharField(max_length=10)),
+                ('standardPath', models.CharField(max_length=200)),
+                ('revision', models.BooleanField()),
+                ('paragraph', models.TextField()),
+            ],
+        ),
+    ]

App/migrations/0002_standards_standardlevel.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-05-24 17:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='standards',
+            name='standardLevel',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

App/migrations/0003_rename_standards_standardstable.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.0.4 on 2022-05-24 17:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0002_standards_standardlevel'),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name='standards',
+            new_name='StandardsTable',
+        ),
+    ]

App/migrations/0004_delete_standardstable.py

@@ -0,0 +1,16 @@
+# Generated by Django 4.0.4 on 2022-05-24 17:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0003_rename_standards_standardstable'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='StandardsTable',
+        ),
+    ]

App/migrations/0005_initial.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.0.4 on 2022-05-24 17:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('App', '0004_delete_standardstable'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Standards',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('standardLevel', models.IntegerField(blank=True, null=True)),
+                ('standardTitle', models.CharField(max_length=200)),
+                ('stamdardNumber', models.CharField(max_length=10)),
+                ('standardPath', models.CharField(max_length=200)),
+                ('revision', models.BooleanField()),
+                ('paragraph', models.TextField()),
+            ],
+        ),
+    ]

App/migrations/0006_alter_standards_paragraph.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-05-24 17:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0005_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='standards',
+            name='paragraph',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

App/migrations/0007_alter_standards_standardpath.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-05-25 18:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0006_alter_standards_paragraph'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='standards',
+            name='standardPath',
+            field=models.CharField(blank=True, max_length=200, null=True),
+        ),
+    ]

App/migrations/0008_level.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.0.4 on 2022-06-01 10:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0007_alter_standards_standardpath'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Level',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('levelNumber', models.IntegerField(blank=True, null=True)),
+                ('levelName', models.CharField(max_length=200)),
+            ],
+        ),
+    ]

App/migrations/0009_alter_standards_standardlevel.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.0.4 on 2022-06-01 10:06
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0008_level'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='standards',
+            name='standardLevel',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='App.level'),
+        ),
+    ]

App/migrations/0010_standards_levelid_alter_standards_standardlevel.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.0.4 on 2022-06-01 10:11
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0009_alter_standards_standardlevel'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='standards',
+            name='levelID',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='App.level'),
+        ),
+        migrations.AlterField(
+            model_name='standards',
+            name='standardLevel',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

App/migrations/0011_alter_level_levelnumber.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.0.4 on 2022-06-01 10:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0010_standards_levelid_alter_standards_standardlevel'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='level',
+            name='levelNumber',
+            field=models.IntegerField(default=1),
+            preserve_default=False,
+        ),
+    ]

App/migrations/0012_searchresults.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.0.4 on 2022-06-01 13:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0011_alter_level_levelnumber'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SearchResults',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('index', models.IntegerField(blank=True, null=True)),
+                ('level', models.IntegerField(blank=True, null=True)),
+                ('levelName', models.CharField(max_length=200)),
+                ('title', models.CharField(max_length=200)),
+                ('paragraph', models.TextField(blank=True, null=True)),
+                ('color', models.CharField(max_length=10)),
+                ('module', models.CharField(max_length=200)),
+                ('heading1', models.CharField(max_length=200)),
+                ('heading2', models.CharField(max_length=200)),
+                ('heading3', models.CharField(max_length=200)),
+                ('heading4', models.CharField(max_length=200)),
+                ('pageNumber', models.IntegerField(blank=True, null=True)),
+            ],
+        ),
+    ]

App/migrations/0013_searchresults_user.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-06-02 11:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0012_searchresults'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='searchresults',
+            name='user',
+            field=models.CharField(blank=True, max_length=17, null=True),
+        ),
+    ]

App/migrations/0014_rename_user_searchresults_session_key.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-06-02 23:04
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0013_searchresults_user'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='searchresults',
+            old_name='user',
+            new_name='session_key',
+        ),
+    ]

App/migrations/0015_searchresults_sentence.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-06-09 10:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0014_rename_user_searchresults_session_key'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='searchresults',
+            name='sentence',
+            field=models.CharField(blank=True, max_length=500, null=True),
+        ),
+    ]

App/migrations/0016_searchresults_session_expiry_date_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.0.4 on 2022-06-10 14:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0015_searchresults_sentence'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='searchresults',
+            name='session_expiry_date',
+            field=models.DateField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='searchresults',
+            name='uniqueID',
+            field=models.CharField(blank=True, max_length=36, null=True),
+        ),
+    ]

App/migrations/0017_alter_searchresults_uniqueid.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-06-10 15:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('App', '0016_searchresults_session_expiry_date_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='searchresults',
+            name='uniqueID',
+            field=models.CharField(blank=True, max_length=200, null=True),
+        ),
+    ]

App/models.py

@@ -0,0 +1,45 @@
+
+from enum import unique
+from django.db import models
+
+# Create your models here.
+
+class Level(models.Model):
+    levelNumber = models.IntegerField()
+    levelName = models.CharField(max_length=200)
+
+    def __str__(self):
+        return str(self.levelNumber)
+
+class Standards(models.Model):
+    standardLevel = models.IntegerField(blank=True, null=True)
+    standardTitle = models.CharField(max_length=200)
+    stamdardNumber = models.CharField(max_length=10)
+    standardPath = models.CharField(max_length=200,blank=True,null=True)
+    revision = models.BooleanField()
+    paragraph = models.TextField(blank=True,null=True)
+    levelID = models.ForeignKey(Level, on_delete=models.CASCADE, blank=True, null=True)
+
+    def __str__(self):
+        return self.stamdardNumber
+
+class SearchResults(models.Model):
+    session_key = models.CharField(max_length=17,blank=True, null=True)
+    uniqueID = models.CharField(max_length=200,blank=True, null=True)
+    session_expiry_date = models.DateField(blank=True, null=True)
+    index = models.IntegerField(blank=True, null=True)
+    level = models.IntegerField(blank=True, null=True)
+    levelName = models.CharField(max_length=200)
+    title = models.CharField(max_length=200)
+    paragraph = models.TextField(blank=True,null=True)
+    color = models.CharField(max_length=10)
+    module = models.CharField(max_length=200)
+    heading1 = models.CharField(max_length=200)
+    heading2 = models.CharField(max_length=200)
+    heading3 = models.CharField(max_length=200)
+    heading4 = models.CharField(max_length=200)
+    pageNumber = models.IntegerField(blank=True, null=True)
+    sentence = models.CharField(max_length=500,blank=True,null=True)
+
+    def __str__(self):
+        return self.heading1

App/similarity.py

@@ -0,0 +1,63 @@
+import json
+import pandas as pd
+#library that contains punctuation
+import string
+#importing nlp library
+import nltk
+import re
+import spacy
+from nltk.stem import WordNetLemmatizer
+
+
+wordnet_lemmatizer = WordNetLemmatizer()
+stopwords = nltk.corpus.stopwords.words('english')
+nlp = spacy.load("en_core_web_lg")
+
+
+
+def phrase_preprocessing(phrase):
+    phrase_punctuationfree="".join([i for i in phrase if i not in string.punctuation])
+    phrase_lower= phrase_punctuationfree.lower()
+    phrase_tokens = re.split('\W',phrase_lower)
+    phrase_nostopwords= [i for i in phrase_tokens if i not in stopwords]
+    phrase_lemma = [wordnet_lemmatizer.lemmatize(word) for word in phrase_nostopwords]
+    new_phrase = ' '.join(phrase_lemma)
+    new_nlp_phrase = nlp(new_phrase)
+    print(new_nlp_phrase)
+    return new_nlp_phrase
+
+def similarityCheck(data, phrase):
+    results = pd.DataFrame(columns=['index','score','sentence', 'paragraph','color','module','level', 'heading1', 'heading2', 'heading3', 'heading4', 'pageNum'])
+
+    i = 1
+    list_of_indices = []
+    for index, row in data.iterrows():    
+        doc = nlp(row['new_sentence'])
+        sim = phrase.similarity(doc)
+        if row['Index'] not in list_of_indices:
+            results.loc[i, 'index'] = index
+            results.loc[i, 'score'] = sim
+            results.loc[i, 'sentence'] = row['Sentences']
+            results.loc[i, 'paragraph'] = row['Paragraph']
+            results.loc[i, 'color'] = row['Color']
+            results.loc[i, 'module'] = row['Module']
+            results.loc[i, 'level'] = row['Level']
+            results.loc[i, 'heading1'] =row['Heading1']
+            results.loc[i, 'heading2'] =row['Heading2']
+            results.loc[i, 'heading3'] =row['Heading3']
+            results.loc[i, 'heading4'] =row['Heading4']
+            results.loc[i, 'pageNum'] =row['PageNum']
+            i+=1
+
+            list_of_indices.append(row['Index'])
+        else:
+            pass
+            
+    results.sort_values(by=['score'],ascending=False,inplace=True)
+    results = results[0:20]
+    results_records = results.reset_index().to_json(orient ='records')
+    results_records = json.loads(results_records)
+    
+
+    return results_records
+